From c841758aa02fb1f3009035d28a05bd865e53d460 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 10:43:05 -0700 Subject: [PATCH 01/88] Add architecture to desktop integration test logs. Add custom runner for arm64 mac support. Enable x86 Linux/Windows tests. --- .github/workflows/integration_tests.yml | 207 ++++++++++++++++++++-- scripts/gha/build_testapps.py | 16 +- scripts/gha/print_matrix_configuration.py | 2 + 3 files changed, 206 insertions(+), 19 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 787031e30..4d4df72ef 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -46,7 +46,11 @@ env: pythonVersion: '3.7' artifactRetentionDays: 2 GITHUB_TOKEN: ${{ github.token }} - + # All self-hosted ARM Mac runners should have this label. Due to how + # our custom reporting works, it must be exactly two words separated + # by a hyphen. The first word must be "macos". The second word will + # be omitted from the summary log. + runnerLabelMacArm64: 'macos-m1custom' jobs: check_and_prepare: runs-on: ubuntu-latest @@ -56,6 +60,9 @@ jobs: pr_number: ${{ steps.set_outputs.outputs.pr_number }} matrix_platform: ${{ steps.matrix_config.outputs.matrix_platform }} matrix_os: ${{ steps.matrix_config.outputs.matrix_os }} + matrix_arch_macos: ${{ steps.matrix_config.outputs.matrix_arch_macos }} + matrix_arch_windows_linux: ${{ steps.matrix_config.outputs.matrix_arch_windows_linux }} + matrix_arch_combined: ${{ steps.matrix_config.outputs.matrix_arch_combined }} matrix_ssl: ${{ steps.matrix_config.outputs.matrix_ssl }} apis: ${{ steps.matrix_config.outputs.apis }} mobile_test_on: ${{ steps.matrix_config.outputs.mobile_test_on }} @@ -63,6 +70,8 @@ jobs: xcode_version: ${{ steps.matrix_config.outputs.xcode_version }} ios_device: ${{ steps.matrix_config.outputs.ios_device }} tvos_device: ${{ steps.matrix_config.outputs.tvos_device }} + # Copy the runner label here because matrix specifiers cannot see env. + runner_label_mac_arm64: ${{ env.runnerLabelMacArm64 }} steps: ### Fail the workflow if the user does not have admin access to run the tests. - name: Check if user has permission to trigger tests @@ -183,11 +192,20 @@ jobs: echo "::set-output name=apis::${apis}" echo "::set-output name=matrix_platform::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k platform -o "${{github.event.inputs.platforms}}" --apis ${apis} )" echo "::set-output name=matrix_os::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k os -o "${{github.event.inputs.operating_systems}}")" - # If building against a packaged SDK, only use boringssl. + echo "::set-output name=matrix_arch_macos::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k architecture_macos)" + echo "::set-output name=matrix_arch_windows_linux::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k architecture_windows_linux)" + # Combine architecture_macos and architecture_windows_linux to get a list of all architectures for the build matrix. + matrix_arch_combined=`echo $( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k architecture_macos | sed 's/[]\[,]//g') \ + $( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k architecture_windows_linux | sed 's/[]\[,]//g' ) \ + | sed 's/ /\n/g' | sort | uniq` + matrix_arch_combined_json=["$(echo ${matrix_arch_combined} | sed 's/ /,/g')"] + echo "::set-output name=matrix_arch_combined::${matrix_arch_combined_json}" + # If building against a packaged SDK, consider it as using boringssl, as the packaged SDK uses boringssl under the hood. + # This avoids trying to install openssl on the system when compiling against the packaged SDK. + # As an added bonus, we ensure that the packaged SDK works even without openssl explicitly installed. if [[ -n "${{ github.event.inputs.test_packaged_sdk }}" ]]; then echo "::warning ::Downloading SDK package from previous run: https://github.com/${{github.repository}}/actions/runs/${{ github.event.inputs.test_packaged_sdk }}" - # Because the mobile tests require this value to be set to 'openssl', set it to 'openssl' here. It won't actually matter later. - echo "::set-output name=matrix_ssl::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k ssl_lib -o openssl )" + echo "::set-output name=matrix_ssl::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k ssl_lib -o boringssl )" else echo "::set-output name=matrix_ssl::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k ssl_lib -o "${{github.event.inputs.desktop_ssl_variants}}" )" fi @@ -209,7 +227,7 @@ jobs: --run_id ${{github.run_id}} build_desktop: - name: build-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }} + name: build-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} needs: [check_and_prepare] runs-on: ${{ matrix.os }} # Skip this if there is an empty matrix (which can happen if "auto" was set above). @@ -219,8 +237,25 @@ jobs: fail-fast: false matrix: os: ${{ fromJson(needs.check_and_prepare.outputs.matrix_os) }} + arch: ${{ fromJson(needs.check_and_prepare.outputs.matrix_arch_combined) }} ssl_variant: ${{ fromJson(needs.check_and_prepare.outputs.matrix_ssl) }} - steps: + # Because matrix_arch_combined combines mac, linux, and windows, we need a few exclusions. + exclude: + # Do not attempt to build for arm64 on Windows or Linux. + - os: windows-latest + arch: arm64 + - os: ubuntu-latest + arch: arm64 + # Do not attempt to build for x86 on Mac. + - os: macos-12 + arch: x86 + # Until we support building openssl from source, we can't link to system openssl when cross-compiling. + # Builds all happen on x64 machines, so arm64 and x86 are technically cross-compiling. + - ssl_variant: openssl + arch: arm64 + - ssl_variant: openssl + arch: x86 + steps: - name: setup Xcode version (macos) if: runner.os == 'macOS' run: sudo xcode-select -s /Applications/Xcode_${{ fromJson(needs.check_and_prepare.outputs.xcode_version)[0] }}.app/Contents/Developer @@ -335,7 +370,7 @@ jobs: python scripts/gha/build_testapps.py --p Desktop \ --t ${{ needs.check_and_prepare.outputs.apis }} \ --output_directory "${{ github.workspace }}" \ - --artifact_name "desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}" \ + --artifact_name "desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}" \ --noadd_timestamp \ --short_output_paths \ ${additional_flags[*]} @@ -343,28 +378,28 @@ jobs: if: ${{ !cancelled() }} shell: bash run: | - if [ ! -f build-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}.log.json ]; then + if [ ! -f build-results-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}.log.json ]; then # No summary was created, make a placeholder one. - echo "__SUMMARY_MISSING__" > build-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}.log.json + echo "__SUMMARY_MISSING__" > build-results-desktop-${{ matrix.os }}-${{ matrix.arch}}-${{ matrix.ssl_variant }}.log.json fi - name: Upload Desktop integration tests artifact uses: actions/upload-artifact@v3 if: ${{ !cancelled() }} with: - name: testapps-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }} - path: testapps-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }} + name: testapps-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} + path: testapps-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} retention-days: ${{ env.artifactRetentionDays }} - name: Upload Desktop build results artifact uses: actions/upload-artifact@v3 if: ${{ !cancelled() }} with: name: log-artifact - path: build-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}* + path: build-results-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}* retention-days: ${{ env.artifactRetentionDays }} - name: Cleanup Local Copies of Uploaded Artifacts shell: bash run: | - rm -rf testapps-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }} + rm -rf testapps-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} - name: Set CLOUDSDK_PYTHON (Windows) shell: bash if: startsWith(matrix.os, 'windows') && !cancelled() @@ -398,7 +433,7 @@ jobs: if: ${{ !cancelled() }} shell: bash run: | - cat build-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}.log + cat build-results-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}.log if [[ "${{ job.status }}" != "success" ]]; then exit 1 fi @@ -859,6 +894,148 @@ jobs: exit 1 fi + # Run tests that depend on custom (self-hosted) runners. + # For now, this is only used for ARM Mac builds. + test_desktop_custom_runners: + name: test-desktop-selfhosted-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} + needs: [check_and_prepare, build_desktop] + runs-on: [self-hosted, firebase-cpp, '${{ matrix.runner_label }}' ] + if: | + needs.check_and_prepare.outputs.apis != '' && + !cancelled() + strategy: + fail-fast: false + matrix: + ssl_variant: ${{ fromJson(needs.check_and_prepare.outputs.matrix_ssl) }} + os: ${{ fromJson(needs.check_and_prepare.outputs.matrix_os) }} + arch: ${{ fromJson(needs.check_and_prepare.outputs.matrix_combined) }} + runner_label: ['${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }}'] + exclude: + # Mac self-hosted runner shouldn't run Linux or Windows tests. + - os: ubuntu-latest + runner_label: ${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }} + - os: windows-latest + runner_label: ${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }} + # For Mac, don't run x64 tests on arm64 runner. + - os: macos-12 + arch: x64 + runner_label: ${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }} + # Until we support building openssl from source, we can't link to system + # openssl when cross-compiling, so exclude openssl from mac arm64 testing. + - os: macos-12 + ssl_variant: openssl + runner_label: ${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }} + steps: + - name: Clean up previous run + shell: bash + if: ${{ contains(needs.check_and_prepare.outputs.matrix_os, matrix.build_os) }} + run: | + echo "Cleaning up previous run" + rm -rf "${{ github.workspace }}"/* + - uses: actions/checkout@v2 + with: + ref: ${{needs.check_and_prepare.outputs.github_ref}} + - name: Download Desktop integration tests artifact + if: ${{ contains(needs.check_and_prepare.outputs.matrix_os, matrix.build_os) }} + uses: actions/download-artifact@v2.0.8 + with: + path: testapps + name: testapps-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} + # Omit all of the prerequisites steps; we ensure that our self-hosted runners + # are configured with all prereqs already installed. + - name: Create keychain (macOS) + if: ${{ runner.os == 'macOS' }} + shell: bash + run: | + echo "Creating temporary keychain" + # Create a local keychain on Mac: + # Clean up previous temp keychain, if any. + security delete-keychain tmp-keychain 2> /dev/null || true + # Create temp keychain file and unlock it. + # (Avoid passing in -p on command line by using interactive mode.) + # Also set it to default settings so there is no unlock timeout. + security -i < firebase.json + fi + set -x + ${cmd_prefix} firebase emulators:exec --only firestore --project demo-example 'python scripts/gha/desktop_tester.py --testapp_dir testapps --logfile_name "desktop-${{ matrix.os }}-${{ matrix.arch}}-${{ matrix.ssl_variant }}"' + set +x + env: + USE_FIRESTORE_EMULATOR: true + - name: Delete keychain (macOS) + if: ${{ always() && runner.os == 'macOS' }} + shell: bash + run: | + # Remove the local keychain on Mac: + # Set back to the default login keychain. + security list-keychains -d user -s login.keychain + # Delete the temp keychain, if it exists. + security delete-keychain tmp-keychain || true + - name: Prepare results summary artifact + if: ${{ !cancelled() }} + shell: bash + run: | + if [ ! -f testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.arch}}-${{ matrix.ssl_variant }}.log.json ]; then + mkdir -p testapps && echo "__SUMMARY_MISSING__" > testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.arch}}-${{ matrix.ssl_variant }}.log.json + fi + - name: Upload Desktop test results artifact + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v2.2.2 + with: + name: log-artifact + path: testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}* + retention-days: ${{ env.artifactRetentionDays }} + - name: Download log artifacts + if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }} + uses: actions/download-artifact@v2.0.8 + with: + path: test_results + name: log-artifact + - name: Update PR label and comment + if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }} + run: | + python scripts/gha/it_workflow.py --stage progress \ + --token ${{github.token}} \ + --issue_number ${{needs.check_and_prepare.outputs.pr_number}}\ + --actor ${{github.actor}} \ + --commit ${{needs.check_and_prepare.outputs.github_ref}} \ + --run_id ${{github.run_id}} + - name: Summarize test results + if: ${{ !cancelled() }} + shell: bash + run: | + cat testapps/test-results-desktop-${{ matrix.runner_label }}-${{ matrix.ssl_variant }}.log + if [[ "${{ job.status }}" != "success" ]]; then + exit 1 + fi + - name: Clean up after this run + shell: bash + if: ${{ always() }} + run: | + echo "Cleaning up after this run" + rm -rf "${{ github.workspace }}"/* + test_android: name: test-android-${{ matrix.build_os }}-${{ matrix.android_device }} needs: [check_and_prepare, build_android] @@ -1154,7 +1331,7 @@ jobs: summarize_results: name: "summarize-results" - needs: [check_and_prepare, test_desktop, test_android, test_ios, test_tvos] + needs: [check_and_prepare, test_desktop, test_desktop_custom_runners, test_android, test_ios, test_tvos] runs-on: ubuntu-latest if: ${{ !cancelled() }} steps: diff --git a/scripts/gha/build_testapps.py b/scripts/gha/build_testapps.py index 6a3d71785..62f22245e 100644 --- a/scripts/gha/build_testapps.py +++ b/scripts/gha/build_testapps.py @@ -107,7 +107,7 @@ _SUPPORTED_PLATFORMS = (_ANDROID, _IOS, _TVOS, _DESKTOP) # Architecture -_SUPPORTED_ARCHITECTURES = ("x64", "arm64") # TODO: add x86 +_SUPPORTED_ARCHITECTURES = ("x64", "x86", "arm64") # TODO: add x86 # Values for iOS SDK flag (where the iOS app will run) _APPLE_SDK_DEVICE = "real" @@ -175,7 +175,8 @@ flags.DEFINE_string( "arch", "x64", - "(Desktop only) Which architecture to build: x64 (all) or arm64 (Mac only).") + "(Desktop only) Which architecture to build: x64 (all), x86 (Windows/Linux), " + "or arm64 (Mac only).") flags.DEFINE_multi_string( "cmake_flag", None, @@ -205,6 +206,10 @@ "Use short directory names for output paths. Useful to avoid hitting file " "path limits on Windows.") +flags.DEFINE_bool( + "gha_build", False, + "Set to true if this is a GitHub Actions build.") + def main(argv): if len(argv) > 1: raise app.UsageError("Too many command-line arguments.") @@ -247,7 +252,10 @@ def main(argv): if _DESKTOP in platforms and not FLAGS.packaged_sdk: vcpkg_arch = FLAGS.arch installer = os.path.join(repo_dir, "scripts", "gha", "build_desktop.py") - _run([sys.executable, installer, "--vcpkg_step_only", "--arch", vcpkg_arch]) + # --gha_build may be required to enable x86 Linux GitHub workarounds. + additional_flags = ["--gha_build"] if FLAGS.gha_build else [] + _run([sys.executable, installer, "--vcpkg_step_only", "--arch", vcpkg_arch] + + additional_flags) toolchain_file = os.path.join( repo_dir, "external", "vcpkg", "scripts", "buildsystems", "vcpkg.cmake") if utils.is_mac_os() and FLAGS.arch == "arm64": @@ -452,7 +460,7 @@ def _build_desktop(sdk_dir, cmake_flags): cmake_configure_cmd = ["cmake", ".", "-DCMAKE_BUILD_TYPE=Debug", "-DFIREBASE_CPP_SDK_DIR=" + sdk_dir] if utils.is_windows_os(): - cmake_configure_cmd += ["-A", "x64"] + cmake_configure_cmd += ["-A", FLAGS.arch] elif utils.is_mac_os(): # Ensure that correct Mac architecture is built. cmake_configure_cmd += ["-DCMAKE_OSX_ARCHITECTURES=%s" % diff --git a/scripts/gha/print_matrix_configuration.py b/scripts/gha/print_matrix_configuration.py index c1bfa09a1..952f38e3d 100644 --- a/scripts/gha/print_matrix_configuration.py +++ b/scripts/gha/print_matrix_configuration.py @@ -128,6 +128,8 @@ "android_device": ["android_target", "android_latest", "emulator_target", "emulator_latest", "emulator_32bit"], "ios_device": ["ios_min", "ios_target", "ios_latest", "simulator_min", "simulator_target", "simulator_latest"], "tvos_device": ["tvos_simulator"], + "architecture_windows_linux": ["x64", "x86"], + "architecture_macos": ["x64", "arm64"], } }, "config": { From 7fe5e5eb329c443593628884169d10c6bc8df19d Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 10:44:56 -0700 Subject: [PATCH 02/88] Add space --- .github/workflows/integration_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 4d4df72ef..7c3774ec9 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -51,6 +51,7 @@ env: # by a hyphen. The first word must be "macos". The second word will # be omitted from the summary log. runnerLabelMacArm64: 'macos-m1custom' + jobs: check_and_prepare: runs-on: ubuntu-latest From 3ea8355e21a53f089700aac712fc8256b796b6d0 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 10:46:10 -0700 Subject: [PATCH 03/88] Fix? --- .github/workflows/integration_tests.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 7c3774ec9..0e3b56d91 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -46,11 +46,6 @@ env: pythonVersion: '3.7' artifactRetentionDays: 2 GITHUB_TOKEN: ${{ github.token }} - # All self-hosted ARM Mac runners should have this label. Due to how - # our custom reporting works, it must be exactly two words separated - # by a hyphen. The first word must be "macos". The second word will - # be omitted from the summary log. - runnerLabelMacArm64: 'macos-m1custom' jobs: check_and_prepare: From 4fe4b9df8c4720ad1e6ed5ace382cf9ff77be5e6 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 10:48:18 -0700 Subject: [PATCH 04/88] Fix?? --- .github/workflows/integration_tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 0e3b56d91..a074c4ac4 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -46,6 +46,11 @@ env: pythonVersion: '3.7' artifactRetentionDays: 2 GITHUB_TOKEN: ${{ github.token }} + # All self-hosted ARM Mac runners should have this label. Due to how + # our custom reporting works, it must be exactly two words separated + # by a hyphen. The first word must be "macos". The second word will + # be omitted from the summary log. + runnerLabelMacArm64: "macos-m1custom" jobs: check_and_prepare: @@ -56,9 +61,6 @@ jobs: pr_number: ${{ steps.set_outputs.outputs.pr_number }} matrix_platform: ${{ steps.matrix_config.outputs.matrix_platform }} matrix_os: ${{ steps.matrix_config.outputs.matrix_os }} - matrix_arch_macos: ${{ steps.matrix_config.outputs.matrix_arch_macos }} - matrix_arch_windows_linux: ${{ steps.matrix_config.outputs.matrix_arch_windows_linux }} - matrix_arch_combined: ${{ steps.matrix_config.outputs.matrix_arch_combined }} matrix_ssl: ${{ steps.matrix_config.outputs.matrix_ssl }} apis: ${{ steps.matrix_config.outputs.apis }} mobile_test_on: ${{ steps.matrix_config.outputs.mobile_test_on }} @@ -66,8 +68,6 @@ jobs: xcode_version: ${{ steps.matrix_config.outputs.xcode_version }} ios_device: ${{ steps.matrix_config.outputs.ios_device }} tvos_device: ${{ steps.matrix_config.outputs.tvos_device }} - # Copy the runner label here because matrix specifiers cannot see env. - runner_label_mac_arm64: ${{ env.runnerLabelMacArm64 }} steps: ### Fail the workflow if the user does not have admin access to run the tests. - name: Check if user has permission to trigger tests From ccf040f63f2707c3ac1d67cd3a5f1597689942bb Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 11:06:53 -0700 Subject: [PATCH 05/88] Remove custom job --- .github/workflows/integration_tests.yml | 146 +----------------------- 1 file changed, 5 insertions(+), 141 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index a074c4ac4..76d306c82 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -61,6 +61,9 @@ jobs: pr_number: ${{ steps.set_outputs.outputs.pr_number }} matrix_platform: ${{ steps.matrix_config.outputs.matrix_platform }} matrix_os: ${{ steps.matrix_config.outputs.matrix_os }} + matrix_arch_macos: ${{ steps.matrix_config.outputs.matrix_arch_macos }} + matrix_arch_windows_linux: ${{ steps.matrix_config.outputs.matrix_arch_windows_linux }} + matrix_arch_combined: ${{ steps.matrix_config.outputs.matrix_arch_combined }} matrix_ssl: ${{ steps.matrix_config.outputs.matrix_ssl }} apis: ${{ steps.matrix_config.outputs.apis }} mobile_test_on: ${{ steps.matrix_config.outputs.mobile_test_on }} @@ -68,6 +71,8 @@ jobs: xcode_version: ${{ steps.matrix_config.outputs.xcode_version }} ios_device: ${{ steps.matrix_config.outputs.ios_device }} tvos_device: ${{ steps.matrix_config.outputs.tvos_device }} + # Copy the runner label here because matrix specifiers cannot see env. + runner_label_mac_arm64: ${{ env.runnerLabelMacArm64 }} steps: ### Fail the workflow if the user does not have admin access to run the tests. - name: Check if user has permission to trigger tests @@ -890,147 +895,6 @@ jobs: exit 1 fi - # Run tests that depend on custom (self-hosted) runners. - # For now, this is only used for ARM Mac builds. - test_desktop_custom_runners: - name: test-desktop-selfhosted-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} - needs: [check_and_prepare, build_desktop] - runs-on: [self-hosted, firebase-cpp, '${{ matrix.runner_label }}' ] - if: | - needs.check_and_prepare.outputs.apis != '' && - !cancelled() - strategy: - fail-fast: false - matrix: - ssl_variant: ${{ fromJson(needs.check_and_prepare.outputs.matrix_ssl) }} - os: ${{ fromJson(needs.check_and_prepare.outputs.matrix_os) }} - arch: ${{ fromJson(needs.check_and_prepare.outputs.matrix_combined) }} - runner_label: ['${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }}'] - exclude: - # Mac self-hosted runner shouldn't run Linux or Windows tests. - - os: ubuntu-latest - runner_label: ${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }} - - os: windows-latest - runner_label: ${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }} - # For Mac, don't run x64 tests on arm64 runner. - - os: macos-12 - arch: x64 - runner_label: ${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }} - # Until we support building openssl from source, we can't link to system - # openssl when cross-compiling, so exclude openssl from mac arm64 testing. - - os: macos-12 - ssl_variant: openssl - runner_label: ${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }} - steps: - - name: Clean up previous run - shell: bash - if: ${{ contains(needs.check_and_prepare.outputs.matrix_os, matrix.build_os) }} - run: | - echo "Cleaning up previous run" - rm -rf "${{ github.workspace }}"/* - - uses: actions/checkout@v2 - with: - ref: ${{needs.check_and_prepare.outputs.github_ref}} - - name: Download Desktop integration tests artifact - if: ${{ contains(needs.check_and_prepare.outputs.matrix_os, matrix.build_os) }} - uses: actions/download-artifact@v2.0.8 - with: - path: testapps - name: testapps-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} - # Omit all of the prerequisites steps; we ensure that our self-hosted runners - # are configured with all prereqs already installed. - - name: Create keychain (macOS) - if: ${{ runner.os == 'macOS' }} - shell: bash - run: | - echo "Creating temporary keychain" - # Create a local keychain on Mac: - # Clean up previous temp keychain, if any. - security delete-keychain tmp-keychain 2> /dev/null || true - # Create temp keychain file and unlock it. - # (Avoid passing in -p on command line by using interactive mode.) - # Also set it to default settings so there is no unlock timeout. - security -i < firebase.json - fi - set -x - ${cmd_prefix} firebase emulators:exec --only firestore --project demo-example 'python scripts/gha/desktop_tester.py --testapp_dir testapps --logfile_name "desktop-${{ matrix.os }}-${{ matrix.arch}}-${{ matrix.ssl_variant }}"' - set +x - env: - USE_FIRESTORE_EMULATOR: true - - name: Delete keychain (macOS) - if: ${{ always() && runner.os == 'macOS' }} - shell: bash - run: | - # Remove the local keychain on Mac: - # Set back to the default login keychain. - security list-keychains -d user -s login.keychain - # Delete the temp keychain, if it exists. - security delete-keychain tmp-keychain || true - - name: Prepare results summary artifact - if: ${{ !cancelled() }} - shell: bash - run: | - if [ ! -f testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.arch}}-${{ matrix.ssl_variant }}.log.json ]; then - mkdir -p testapps && echo "__SUMMARY_MISSING__" > testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.arch}}-${{ matrix.ssl_variant }}.log.json - fi - - name: Upload Desktop test results artifact - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v2.2.2 - with: - name: log-artifact - path: testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}* - retention-days: ${{ env.artifactRetentionDays }} - - name: Download log artifacts - if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }} - uses: actions/download-artifact@v2.0.8 - with: - path: test_results - name: log-artifact - - name: Update PR label and comment - if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }} - run: | - python scripts/gha/it_workflow.py --stage progress \ - --token ${{github.token}} \ - --issue_number ${{needs.check_and_prepare.outputs.pr_number}}\ - --actor ${{github.actor}} \ - --commit ${{needs.check_and_prepare.outputs.github_ref}} \ - --run_id ${{github.run_id}} - - name: Summarize test results - if: ${{ !cancelled() }} - shell: bash - run: | - cat testapps/test-results-desktop-${{ matrix.runner_label }}-${{ matrix.ssl_variant }}.log - if [[ "${{ job.status }}" != "success" ]]; then - exit 1 - fi - - name: Clean up after this run - shell: bash - if: ${{ always() }} - run: | - echo "Cleaning up after this run" - rm -rf "${{ github.workspace }}"/* test_android: name: test-android-${{ matrix.build_os }}-${{ matrix.android_device }} From b9180dfa3cf322b424f4b9332b7e1348fc00b67a Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 11:07:40 -0700 Subject: [PATCH 06/88] Restore old integration test workflow --- .github/workflows/integration_tests.yml | 65 ++++++------------------- 1 file changed, 14 insertions(+), 51 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 76d306c82..787031e30 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -46,11 +46,6 @@ env: pythonVersion: '3.7' artifactRetentionDays: 2 GITHUB_TOKEN: ${{ github.token }} - # All self-hosted ARM Mac runners should have this label. Due to how - # our custom reporting works, it must be exactly two words separated - # by a hyphen. The first word must be "macos". The second word will - # be omitted from the summary log. - runnerLabelMacArm64: "macos-m1custom" jobs: check_and_prepare: @@ -61,9 +56,6 @@ jobs: pr_number: ${{ steps.set_outputs.outputs.pr_number }} matrix_platform: ${{ steps.matrix_config.outputs.matrix_platform }} matrix_os: ${{ steps.matrix_config.outputs.matrix_os }} - matrix_arch_macos: ${{ steps.matrix_config.outputs.matrix_arch_macos }} - matrix_arch_windows_linux: ${{ steps.matrix_config.outputs.matrix_arch_windows_linux }} - matrix_arch_combined: ${{ steps.matrix_config.outputs.matrix_arch_combined }} matrix_ssl: ${{ steps.matrix_config.outputs.matrix_ssl }} apis: ${{ steps.matrix_config.outputs.apis }} mobile_test_on: ${{ steps.matrix_config.outputs.mobile_test_on }} @@ -71,8 +63,6 @@ jobs: xcode_version: ${{ steps.matrix_config.outputs.xcode_version }} ios_device: ${{ steps.matrix_config.outputs.ios_device }} tvos_device: ${{ steps.matrix_config.outputs.tvos_device }} - # Copy the runner label here because matrix specifiers cannot see env. - runner_label_mac_arm64: ${{ env.runnerLabelMacArm64 }} steps: ### Fail the workflow if the user does not have admin access to run the tests. - name: Check if user has permission to trigger tests @@ -193,20 +183,11 @@ jobs: echo "::set-output name=apis::${apis}" echo "::set-output name=matrix_platform::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k platform -o "${{github.event.inputs.platforms}}" --apis ${apis} )" echo "::set-output name=matrix_os::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k os -o "${{github.event.inputs.operating_systems}}")" - echo "::set-output name=matrix_arch_macos::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k architecture_macos)" - echo "::set-output name=matrix_arch_windows_linux::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k architecture_windows_linux)" - # Combine architecture_macos and architecture_windows_linux to get a list of all architectures for the build matrix. - matrix_arch_combined=`echo $( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k architecture_macos | sed 's/[]\[,]//g') \ - $( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k architecture_windows_linux | sed 's/[]\[,]//g' ) \ - | sed 's/ /\n/g' | sort | uniq` - matrix_arch_combined_json=["$(echo ${matrix_arch_combined} | sed 's/ /,/g')"] - echo "::set-output name=matrix_arch_combined::${matrix_arch_combined_json}" - # If building against a packaged SDK, consider it as using boringssl, as the packaged SDK uses boringssl under the hood. - # This avoids trying to install openssl on the system when compiling against the packaged SDK. - # As an added bonus, we ensure that the packaged SDK works even without openssl explicitly installed. + # If building against a packaged SDK, only use boringssl. if [[ -n "${{ github.event.inputs.test_packaged_sdk }}" ]]; then echo "::warning ::Downloading SDK package from previous run: https://github.com/${{github.repository}}/actions/runs/${{ github.event.inputs.test_packaged_sdk }}" - echo "::set-output name=matrix_ssl::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k ssl_lib -o boringssl )" + # Because the mobile tests require this value to be set to 'openssl', set it to 'openssl' here. It won't actually matter later. + echo "::set-output name=matrix_ssl::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k ssl_lib -o openssl )" else echo "::set-output name=matrix_ssl::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k ssl_lib -o "${{github.event.inputs.desktop_ssl_variants}}" )" fi @@ -228,7 +209,7 @@ jobs: --run_id ${{github.run_id}} build_desktop: - name: build-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} + name: build-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }} needs: [check_and_prepare] runs-on: ${{ matrix.os }} # Skip this if there is an empty matrix (which can happen if "auto" was set above). @@ -238,25 +219,8 @@ jobs: fail-fast: false matrix: os: ${{ fromJson(needs.check_and_prepare.outputs.matrix_os) }} - arch: ${{ fromJson(needs.check_and_prepare.outputs.matrix_arch_combined) }} ssl_variant: ${{ fromJson(needs.check_and_prepare.outputs.matrix_ssl) }} - # Because matrix_arch_combined combines mac, linux, and windows, we need a few exclusions. - exclude: - # Do not attempt to build for arm64 on Windows or Linux. - - os: windows-latest - arch: arm64 - - os: ubuntu-latest - arch: arm64 - # Do not attempt to build for x86 on Mac. - - os: macos-12 - arch: x86 - # Until we support building openssl from source, we can't link to system openssl when cross-compiling. - # Builds all happen on x64 machines, so arm64 and x86 are technically cross-compiling. - - ssl_variant: openssl - arch: arm64 - - ssl_variant: openssl - arch: x86 - steps: + steps: - name: setup Xcode version (macos) if: runner.os == 'macOS' run: sudo xcode-select -s /Applications/Xcode_${{ fromJson(needs.check_and_prepare.outputs.xcode_version)[0] }}.app/Contents/Developer @@ -371,7 +335,7 @@ jobs: python scripts/gha/build_testapps.py --p Desktop \ --t ${{ needs.check_and_prepare.outputs.apis }} \ --output_directory "${{ github.workspace }}" \ - --artifact_name "desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}" \ + --artifact_name "desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}" \ --noadd_timestamp \ --short_output_paths \ ${additional_flags[*]} @@ -379,28 +343,28 @@ jobs: if: ${{ !cancelled() }} shell: bash run: | - if [ ! -f build-results-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}.log.json ]; then + if [ ! -f build-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}.log.json ]; then # No summary was created, make a placeholder one. - echo "__SUMMARY_MISSING__" > build-results-desktop-${{ matrix.os }}-${{ matrix.arch}}-${{ matrix.ssl_variant }}.log.json + echo "__SUMMARY_MISSING__" > build-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}.log.json fi - name: Upload Desktop integration tests artifact uses: actions/upload-artifact@v3 if: ${{ !cancelled() }} with: - name: testapps-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} - path: testapps-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} + name: testapps-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }} + path: testapps-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }} retention-days: ${{ env.artifactRetentionDays }} - name: Upload Desktop build results artifact uses: actions/upload-artifact@v3 if: ${{ !cancelled() }} with: name: log-artifact - path: build-results-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}* + path: build-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}* retention-days: ${{ env.artifactRetentionDays }} - name: Cleanup Local Copies of Uploaded Artifacts shell: bash run: | - rm -rf testapps-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} + rm -rf testapps-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }} - name: Set CLOUDSDK_PYTHON (Windows) shell: bash if: startsWith(matrix.os, 'windows') && !cancelled() @@ -434,7 +398,7 @@ jobs: if: ${{ !cancelled() }} shell: bash run: | - cat build-results-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}.log + cat build-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}.log if [[ "${{ job.status }}" != "success" ]]; then exit 1 fi @@ -895,7 +859,6 @@ jobs: exit 1 fi - test_android: name: test-android-${{ matrix.build_os }}-${{ matrix.android_device }} needs: [check_and_prepare, build_android] @@ -1191,7 +1154,7 @@ jobs: summarize_results: name: "summarize-results" - needs: [check_and_prepare, test_desktop, test_desktop_custom_runners, test_android, test_ios, test_tvos] + needs: [check_and_prepare, test_desktop, test_android, test_ios, test_tvos] runs-on: ubuntu-latest if: ${{ !cancelled() }} steps: From be332696807791a98c175cb7f7790554def9b355 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 11:10:07 -0700 Subject: [PATCH 07/88] Bring in a bit --- .github/workflows/integration_tests.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 787031e30..a39d49a44 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -46,6 +46,11 @@ env: pythonVersion: '3.7' artifactRetentionDays: 2 GITHUB_TOKEN: ${{ github.token }} + # All self-hosted ARM Mac runners should have this label. Due to how + # our custom reporting works, it must be exactly two words separated + # by a hyphen. The first word must be "macos". The second word will + # be omitted from the summary log. + runnerLabelMacArm64: "macos-m1custom" jobs: check_and_prepare: @@ -56,6 +61,9 @@ jobs: pr_number: ${{ steps.set_outputs.outputs.pr_number }} matrix_platform: ${{ steps.matrix_config.outputs.matrix_platform }} matrix_os: ${{ steps.matrix_config.outputs.matrix_os }} + matrix_arch_macos: ${{ steps.matrix_config.outputs.matrix_arch_macos }} + matrix_arch_windows_linux: ${{ steps.matrix_config.outputs.matrix_arch_windows_linux }} + matrix_arch_combined: ${{ steps.matrix_config.outputs.matrix_arch_combined }} matrix_ssl: ${{ steps.matrix_config.outputs.matrix_ssl }} apis: ${{ steps.matrix_config.outputs.apis }} mobile_test_on: ${{ steps.matrix_config.outputs.mobile_test_on }} @@ -183,6 +191,14 @@ jobs: echo "::set-output name=apis::${apis}" echo "::set-output name=matrix_platform::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k platform -o "${{github.event.inputs.platforms}}" --apis ${apis} )" echo "::set-output name=matrix_os::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k os -o "${{github.event.inputs.operating_systems}}")" + echo "::set-output name=matrix_arch_macos::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k architecture_macos)" + echo "::set-output name=matrix_arch_windows_linux::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k architecture_windows_linux)" + # Combine architecture_macos and architecture_windows_linux to get a list of all architectures for the build matrix. + matrix_arch_combined=`echo $( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k architecture_macos | sed 's/[]\[,]//g') \ + $( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k architecture_windows_linux | sed 's/[]\[,]//g' ) \ + | sed 's/ /\n/g' | sort | uniq` + matrix_arch_combined_json=["$(echo ${matrix_arch_combined} | sed 's/ /,/g')"] + echo "::set-output name=matrix_arch_combined::${matrix_arch_combined_json}" # If building against a packaged SDK, only use boringssl. if [[ -n "${{ github.event.inputs.test_packaged_sdk }}" ]]; then echo "::warning ::Downloading SDK package from previous run: https://github.com/${{github.repository}}/actions/runs/${{ github.event.inputs.test_packaged_sdk }}" From 9e46459f3fcf0fa21231dfffc5ba7c6999bf8055 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 11:10:48 -0700 Subject: [PATCH 08/88] Add more --- .github/workflows/integration_tests.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index a39d49a44..6e05f1747 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -199,11 +199,12 @@ jobs: | sed 's/ /\n/g' | sort | uniq` matrix_arch_combined_json=["$(echo ${matrix_arch_combined} | sed 's/ /,/g')"] echo "::set-output name=matrix_arch_combined::${matrix_arch_combined_json}" - # If building against a packaged SDK, only use boringssl. + # If building against a packaged SDK, consider it as using boringssl, as the packaged SDK uses boringssl under the hood. + # This avoids trying to install openssl on the system when compiling against the packaged SDK. + # As an added bonus, we ensure that the packaged SDK works even without openssl explicitly installed. if [[ -n "${{ github.event.inputs.test_packaged_sdk }}" ]]; then echo "::warning ::Downloading SDK package from previous run: https://github.com/${{github.repository}}/actions/runs/${{ github.event.inputs.test_packaged_sdk }}" - # Because the mobile tests require this value to be set to 'openssl', set it to 'openssl' here. It won't actually matter later. - echo "::set-output name=matrix_ssl::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k ssl_lib -o openssl )" + echo "::set-output name=matrix_ssl::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k ssl_lib -o boringssl )" else echo "::set-output name=matrix_ssl::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k ssl_lib -o "${{github.event.inputs.desktop_ssl_variants}}" )" fi From 46a5d0ada390dff7db0f1ce243431b87626834b0 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 11:12:16 -0700 Subject: [PATCH 09/88] Add more --- .github/workflows/integration_tests.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 6e05f1747..60520880c 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -236,7 +236,23 @@ jobs: fail-fast: false matrix: os: ${{ fromJson(needs.check_and_prepare.outputs.matrix_os) }} + arch: ${{ fromJson(needs.check_and_prepare.outputs.matrix_arch_combined) }} ssl_variant: ${{ fromJson(needs.check_and_prepare.outputs.matrix_ssl) }} + exclude: + # Do not attempt to build for arm64 on Windows or Linux. + - os: windows-latest + arch: arm64 + - os: ubuntu-latest + arch: arm64 + # Do not attempt to build for x86 on Mac. + - os: macos-12 + arch: x86 + # Until we support building openssl from source, we can't link to system openssl when cross-compiling. + # Builds all happen on x64 machines, so arm64 and x86 are technically cross-compiling. + - ssl_variant: openssl + arch: arm64 + - ssl_variant: openssl + arch: x86 steps: - name: setup Xcode version (macos) if: runner.os == 'macOS' From 9d02df8275d0d75a4b6143fd984c639d78f16b24 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 11:13:35 -0700 Subject: [PATCH 10/88] Reindent --- .github/workflows/integration_tests.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 60520880c..5849987fc 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -226,7 +226,7 @@ jobs: --run_id ${{github.run_id}} build_desktop: - name: build-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }} + name: build-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} needs: [check_and_prepare] runs-on: ${{ matrix.os }} # Skip this if there is an empty matrix (which can happen if "auto" was set above). @@ -238,17 +238,18 @@ jobs: os: ${{ fromJson(needs.check_and_prepare.outputs.matrix_os) }} arch: ${{ fromJson(needs.check_and_prepare.outputs.matrix_arch_combined) }} ssl_variant: ${{ fromJson(needs.check_and_prepare.outputs.matrix_ssl) }} + # Because matrix_arch_combined combines mac, linux, and windows, we need a few exclusions. exclude: # Do not attempt to build for arm64 on Windows or Linux. - os: windows-latest arch: arm64 - os: ubuntu-latest arch: arm64 - # Do not attempt to build for x86 on Mac. + # Do not attempt to build for x86 on Mac. - os: macos-12 arch: x86 - # Until we support building openssl from source, we can't link to system openssl when cross-compiling. - # Builds all happen on x64 machines, so arm64 and x86 are technically cross-compiling. + # Until we support building openssl from source, we can't link to system openssl when cross-compiling. + # Builds all happen on x64 machines, so arm64 and x86 are technically cross-compiling. - ssl_variant: openssl arch: arm64 - ssl_variant: openssl From aea910c9810dfddbb421888b57f160caf53b6015 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 11:18:34 -0700 Subject: [PATCH 11/88] Add arch to log --- .github/workflows/integration_tests.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 5849987fc..4e910ca26 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -369,7 +369,7 @@ jobs: python scripts/gha/build_testapps.py --p Desktop \ --t ${{ needs.check_and_prepare.outputs.apis }} \ --output_directory "${{ github.workspace }}" \ - --artifact_name "desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}" \ + --artifact_name "desktop-${{ matrix.os }}-${{ matrix.arch}}-${{ matrix.ssl_variant }}" \ --noadd_timestamp \ --short_output_paths \ ${additional_flags[*]} @@ -377,28 +377,28 @@ jobs: if: ${{ !cancelled() }} shell: bash run: | - if [ ! -f build-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}.log.json ]; then + if [ ! -f build-results-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}.log.json ]; then # No summary was created, make a placeholder one. - echo "__SUMMARY_MISSING__" > build-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}.log.json + echo "__SUMMARY_MISSING__" > build-results-desktop-${{ matrix.os }}-${{ matrix.arch}}-${{ matrix.ssl_variant }}.log.json fi - name: Upload Desktop integration tests artifact uses: actions/upload-artifact@v3 if: ${{ !cancelled() }} with: - name: testapps-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }} - path: testapps-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }} + name: testapps-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} + path: testapps-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} retention-days: ${{ env.artifactRetentionDays }} - name: Upload Desktop build results artifact uses: actions/upload-artifact@v3 if: ${{ !cancelled() }} with: name: log-artifact - path: build-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}* + path: build-results-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}* retention-days: ${{ env.artifactRetentionDays }} - name: Cleanup Local Copies of Uploaded Artifacts shell: bash run: | - rm -rf testapps-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }} + rm -rf testapps-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} - name: Set CLOUDSDK_PYTHON (Windows) shell: bash if: startsWith(matrix.os, 'windows') && !cancelled() @@ -432,7 +432,7 @@ jobs: if: ${{ !cancelled() }} shell: bash run: | - cat build-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}.log + cat build-results-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}.log if [[ "${{ job.status }}" != "success" ]]; then exit 1 fi From 2eaf0e9e66123c307c5c5f9f59b04c0f884bd094 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 11:19:16 -0700 Subject: [PATCH 12/88] Add label to outputs --- .github/workflows/integration_tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 4e910ca26..35b3dbd34 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -71,6 +71,8 @@ jobs: xcode_version: ${{ steps.matrix_config.outputs.xcode_version }} ios_device: ${{ steps.matrix_config.outputs.ios_device }} tvos_device: ${{ steps.matrix_config.outputs.tvos_device }} + # Copy the runner label here because matrix specifiers cannot see env. + runner_label_mac_arm64: ${{ env.runnerLabelMacArm64 }} steps: ### Fail the workflow if the user does not have admin access to run the tests. - name: Check if user has permission to trigger tests From f49ed076e7e990f02700528fb134cf68b2740f59 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 11:22:16 -0700 Subject: [PATCH 13/88] Add selfhosted job (just the basics) --- .github/workflows/integration_tests.yml | 43 +++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 35b3dbd34..ee6bf12a8 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -371,7 +371,7 @@ jobs: python scripts/gha/build_testapps.py --p Desktop \ --t ${{ needs.check_and_prepare.outputs.apis }} \ --output_directory "${{ github.workspace }}" \ - --artifact_name "desktop-${{ matrix.os }}-${{ matrix.arch}}-${{ matrix.ssl_variant }}" \ + --artifact_name "desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}" \ --noadd_timestamp \ --short_output_paths \ ${additional_flags[*]} @@ -381,7 +381,7 @@ jobs: run: | if [ ! -f build-results-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}.log.json ]; then # No summary was created, make a placeholder one. - echo "__SUMMARY_MISSING__" > build-results-desktop-${{ matrix.os }}-${{ matrix.arch}}-${{ matrix.ssl_variant }}.log.json + echo "__SUMMARY_MISSING__" > build-results-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}.log.json fi - name: Upload Desktop integration tests artifact uses: actions/upload-artifact@v3 @@ -895,6 +895,45 @@ jobs: exit 1 fi + # Run tests that depend on custom (self-hosted) runners. + # For now, this is only used for ARM Mac builds. + test_desktop_custom_runners: + name: test-desktop-selfhosted-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} + needs: [check_and_prepare, build_desktop] + runs-on: [self-hosted, firebase-cpp, '${{ matrix.runner_label }}' ] + if: needs.check_and_prepare.outputs.apis != '' && !cancelled() + strategy: + fail-fast: false + matrix: + ssl_variant: ${{ fromJson(needs.check_and_prepare.outputs.matrix_ssl) }} + os: ${{ fromJson(needs.check_and_prepare.outputs.matrix_os) }} + arch: ${{ fromJson(needs.check_and_prepare.outputs.matrix_combined) }} + runner_label: ['${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }}'] + exclude: + # Mac self-hosted runner shouldn't run Linux or Windows tests. + - os: ubuntu-latest + runner_label: ${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }} + - os: windows-latest + runner_label: ${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }} + # For Mac, don't run x64 tests on arm64 runner. + - os: macos-12 + arch: x64 + runner_label: ${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }} + # Until we support building openssl from source, we can't link to system + # openssl when cross-compiling, so exclude openssl from mac arm64 testing. + - os: macos-12 + ssl_variant: openssl + runner_label: ${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }} + steps: + - name: Clean up previous run + shell: bash + if: ${{ contains(needs.check_and_prepare.outputs.matrix_os, matrix.build_os) }} + run: | + echo "Cleaning up previous run" + rm -rf "${{ github.workspace }}"/* + + + test_android: name: test-android-${{ matrix.build_os }}-${{ matrix.android_device }} needs: [check_and_prepare, build_android] From ff26aa88db41acf8739a2bab5efc5a5c2cf91ecd Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 11:23:59 -0700 Subject: [PATCH 14/88] Add the rest of the selfhosted job --- .github/workflows/integration_tests.yml | 103 +++++++++++++++++++++++- 1 file changed, 102 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index ee6bf12a8..de858490f 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -927,10 +927,111 @@ jobs: steps: - name: Clean up previous run shell: bash - if: ${{ contains(needs.check_and_prepare.outputs.matrix_os, matrix.build_os) }} run: | echo "Cleaning up previous run" rm -rf "${{ github.workspace }}"/* + - uses: actions/checkout@v2 + with: + ref: ${{needs.check_and_prepare.outputs.github_ref}} + - name: Download Desktop integration tests artifact + uses: actions/download-artifact@v2.0.8 + with: + path: testapps + name: testapps-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} + # Omit all of the prerequisites steps; we ensure that our self-hosted runners + # are configured with all prereqs already installed. + - name: Create keychain (macOS) + if: ${{ runner.os == 'macOS' }} + shell: bash + run: | + echo "Creating temporary keychain" + # Create a local keychain on Mac: + # Clean up previous temp keychain, if any. + security delete-keychain tmp-keychain 2> /dev/null || true + # Create temp keychain file and unlock it. + # (Avoid passing in -p on command line by using interactive mode.) + # Also set it to default settings so there is no unlock timeout. + security -i < firebase.json + fi + set -x + ${cmd_prefix} firebase emulators:exec --only firestore --project demo-example 'python scripts/gha/desktop_tester.py --testapp_dir testapps --logfile_name "desktop-${{ matrix.os }}-${{ matrix.arch}}-${{ matrix.ssl_variant }}"' + set +x + env: + USE_FIRESTORE_EMULATOR: true + - name: Delete keychain (macOS) + if: ${{ always() && runner.os == 'macOS' }} + shell: bash + run: | + # Remove the local keychain on Mac: + # Set back to the default login keychain. + security list-keychains -d user -s login.keychain + # Delete the temp keychain, if it exists. + security delete-keychain tmp-keychain || true + - name: Prepare results summary artifact + if: ${{ !cancelled() }} + shell: bash + run: | + if [ ! -f testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.arch}}-${{ matrix.ssl_variant }}.log.json ]; then + mkdir -p testapps && echo "__SUMMARY_MISSING__" > testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.arch}}-${{ matrix.ssl_variant }}.log.json + fi + - name: Upload Desktop test results artifact + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v2.2.2 + with: + name: log-artifact + path: testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}* + retention-days: ${{ env.artifactRetentionDays }} + - name: Download log artifacts + if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }} + uses: actions/download-artifact@v2.0.8 + with: + path: test_results + name: log-artifact + - name: Update PR label and comment + if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }} + run: | + python scripts/gha/it_workflow.py --stage progress \ + --token ${{github.token}} \ + --issue_number ${{needs.check_and_prepare.outputs.pr_number}}\ + --actor ${{github.actor}} \ + --commit ${{needs.check_and_prepare.outputs.github_ref}} \ + --run_id ${{github.run_id}} + - name: Summarize test results + if: ${{ !cancelled() }} + shell: bash + run: | + cat testapps/test-results-desktop-${{ matrix.runner_label }}-${{ matrix.ssl_variant }}.log + if [[ "${{ job.status }}" != "success" ]]; then + exit 1 + fi + - name: Clean up after this run + shell: bash + if: ${{ always() }} + run: | + echo "Cleaning up after this run" + rm -rf "${{ github.workspace }}"/* From 237b376f1700ee38c1391a716d302f1d152f7acd Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 11:28:50 -0700 Subject: [PATCH 15/88] Remove stray character, add dependency --- .github/workflows/integration_tests.yml | 2 +- scripts/gha/build_testapps.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index de858490f..482d0f414 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -1330,7 +1330,7 @@ jobs: summarize_results: name: "summarize-results" - needs: [check_and_prepare, test_desktop, test_android, test_ios, test_tvos] + needs: [check_and_prepare, test_desktop, test_desktop_custom_runners, test_android, test_ios, test_tvos] runs-on: ubuntu-latest if: ${{ !cancelled() }} steps: diff --git a/scripts/gha/build_testapps.py b/scripts/gha/build_testapps.py index 62f22245e..8b753fb79 100644 --- a/scripts/gha/build_testapps.py +++ b/scripts/gha/build_testapps.py @@ -208,7 +208,7 @@ flags.DEFINE_bool( "gha_build", False, - "Set to true if this is a GitHub Actions build.") + "Set to true if this is a GitHub Actions build.") def main(argv): if len(argv) > 1: From d0622a37849eb00c09da26116303d7d1fbc7530e Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 11:29:46 -0700 Subject: [PATCH 16/88] Add --gha_build to command line --- .github/workflows/integration_tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 482d0f414..c71bad680 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -374,6 +374,7 @@ jobs: --artifact_name "desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}" \ --noadd_timestamp \ --short_output_paths \ + --gha_build \ ${additional_flags[*]} - name: Prepare results summary artifact if: ${{ !cancelled() }} @@ -521,6 +522,7 @@ jobs: --artifact_name "android-${{ matrix.os }}" \ --noadd_timestamp \ --short_output_paths \ + --gha_build \ ${additional_flags[*]} - name: Prepare results summary artifact if: ${{ !cancelled() }} @@ -640,6 +642,7 @@ jobs: --artifact_name "ios-${{ matrix.os }}" \ --noadd_timestamp \ --short_output_paths \ + --gha_build \ ${additional_flags[*]} - name: Prepare results summary artifact if: ${{ !cancelled() }} @@ -758,6 +761,7 @@ jobs: --artifact_name "tvos-${{ matrix.os }}" \ --noadd_timestamp \ --short_output_paths \ + --gha_build \ ${additional_flags[*]} - name: Prepare results summary artifact if: ${{ !cancelled() }} From cab86a905faa8b00c86228182cbc108160f86ae0 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 11:42:27 -0700 Subject: [PATCH 17/88] Temporarily add arm64 to the quick tests --- scripts/gha/print_matrix_configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gha/print_matrix_configuration.py b/scripts/gha/print_matrix_configuration.py index 952f38e3d..55a7b28ee 100644 --- a/scripts/gha/print_matrix_configuration.py +++ b/scripts/gha/print_matrix_configuration.py @@ -108,7 +108,7 @@ "tvos_device": ["tvos_simulator"], "build_type": ["Debug"], "architecture_windows_linux": ["x64"], - "architecture_macos": ["x64"], + "architecture_macos": ["x64", "arm64"], "msvc_runtime": ["dynamic"], "cpp_compiler_windows": ["VisualStudio2019"], "cpp_compiler_linux": ["clang-11.0"], From 0829d09b414b266fed569033e8583e271399c04e Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 11:45:34 -0700 Subject: [PATCH 18/88] Get rid of placeholder arm64 --- scripts/gha/print_matrix_configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gha/print_matrix_configuration.py b/scripts/gha/print_matrix_configuration.py index 55a7b28ee..952f38e3d 100644 --- a/scripts/gha/print_matrix_configuration.py +++ b/scripts/gha/print_matrix_configuration.py @@ -108,7 +108,7 @@ "tvos_device": ["tvos_simulator"], "build_type": ["Debug"], "architecture_windows_linux": ["x64"], - "architecture_macos": ["x64", "arm64"], + "architecture_macos": ["x64"], "msvc_runtime": ["dynamic"], "cpp_compiler_windows": ["VisualStudio2019"], "cpp_compiler_linux": ["clang-11.0"], From c41505e4c693de45e52da17b07971ccbec1a88fc Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 11:46:15 -0700 Subject: [PATCH 19/88] Remove comment --- scripts/gha/build_testapps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gha/build_testapps.py b/scripts/gha/build_testapps.py index 8b753fb79..8b6f5b5b3 100644 --- a/scripts/gha/build_testapps.py +++ b/scripts/gha/build_testapps.py @@ -107,7 +107,7 @@ _SUPPORTED_PLATFORMS = (_ANDROID, _IOS, _TVOS, _DESKTOP) # Architecture -_SUPPORTED_ARCHITECTURES = ("x64", "x86", "arm64") # TODO: add x86 +_SUPPORTED_ARCHITECTURES = ("x64", "x86", "arm64") # Values for iOS SDK flag (where the iOS app will run) _APPLE_SDK_DEVICE = "real" From ef0423a391a30832092d54279d93a17f17eb71c7 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 11:59:46 -0700 Subject: [PATCH 20/88] Add arch fields to test_desktop as well. --- .github/workflows/integration_tests.yml | 34 +++++++++++++++++-------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index c71bad680..afb32531f 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -809,7 +809,7 @@ jobs: fi test_desktop: - name: test-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }} + name: test-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} needs: [check_and_prepare, build_desktop] runs-on: ${{ matrix.os }} if: contains(needs.check_and_prepare.outputs.matrix_platform, 'Desktop') && needs.check_and_prepare.outputs.apis != '' && !cancelled() @@ -817,7 +817,21 @@ jobs: fail-fast: false matrix: os: ${{ fromJson(needs.check_and_prepare.outputs.matrix_os) }} + arch: ${{ fromJson(needs.check_and_prepare.outputs.matrix_arch_combined) }} ssl_variant: ${{ fromJson(needs.check_and_prepare.outputs.matrix_ssl) }} + # Because matrix_arch_combined combines mac, linux, and windows, we need a few exclusions. + exclude: + # Do not attempt to run for arm64 on Windows or Linux. + - os: windows-latest + arch: arm64 + - os: ubuntu-latest + arch: arm64 + # Do not attempt to run for x86 on Mac. + - os: macos-12 + arch: x86 + # MacOS hosted runner cannot test arm64 code. + - os: macos-12 + arch: arm64 steps: - name: setup Xcode version (macos) if: runner.os == 'macOS' @@ -828,8 +842,8 @@ jobs: - name: Download Desktop integration tests artifact uses: actions/download-artifact@v3 with: - path: testapps/testapps-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }} - name: testapps-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }} + path: testapps/testapps-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} + name: testapps-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} - name: Setup python uses: actions/setup-python@v2 with: @@ -849,31 +863,31 @@ jobs: shell: bash command: | pip install -r scripts/gha/requirements.txt - python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}" --artifact "testapps/testapps-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}" + python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}" --artifact "testapps/testapps-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}" - name: Run Desktop integration tests env: USE_FIRESTORE_EMULATOR: true shell: bash run: | if [[ "${{ needs.check_and_prepare.outputs.apis }}" == *"firestore"* ]]; then - firebase emulators:exec --only firestore --project demo-example 'python scripts/gha/desktop_tester.py --testapp_dir testapps --logfile_name "desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}"' + firebase emulators:exec --only firestore --project demo-example 'python scripts/gha/desktop_tester.py --testapp_dir testapps --logfile_name "desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}"' else - python scripts/gha/desktop_tester.py --testapp_dir testapps --logfile_name "desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}" + python scripts/gha/desktop_tester.py --testapp_dir testapps --logfile_name "desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}" fi - name: Prepare results summary artifact if: ${{ !cancelled() }} shell: bash run: | # If testapps do not exist, then it's a build error not test error. - if [ -d "testapps/testapps-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}" && ! -f testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}.log.json ]; then - mkdir -p testapps && echo "__SUMMARY_MISSING__" > testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}.log.json + if [ -d "testapps/testapps-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}" && ! -f testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}.log.json ]; then + mkdir -p testapps && echo "__SUMMARY_MISSING__" > testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}.log.json fi - name: Upload Desktop test results artifact if: ${{ !cancelled() }} uses: actions/upload-artifact@v3 with: name: log-artifact - path: testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}* + path: testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}* retention-days: ${{ env.artifactRetentionDays }} - name: Download log artifacts if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }} @@ -894,7 +908,7 @@ jobs: if: ${{ !cancelled() }} shell: bash run: | - cat testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}.log + cat testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}.log if [[ "${{ job.status }}" != "success" ]]; then exit 1 fi From f9d4cf4a2845aec13611145ad7b393d023fdb65e Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 12:03:23 -0700 Subject: [PATCH 21/88] Remove trailing spaces --- .github/workflows/integration_tests.yml | 30 ++++++++++++------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index afb32531f..71a0ab2c1 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -83,7 +83,7 @@ jobs: ### trigger value: manual_trigger, scheduled_trigger, label_trigger, postsubmit_trigger - id: set_outputs run: | - if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then if [[ "${{ github.event.inputs.test_pull_request }}" != "nightly-packaging" ]]; then # Triggered manually echo "::set-output name=trigger::manual_trigger" @@ -134,12 +134,12 @@ jobs: if: ${{ !steps.set_outputs.outputs.trigger }} uses: andymckay/cancel-action@0.2 - name: Wait for workflow cancellation - if: ${{ !steps.set_outputs.outputs.trigger }} + if: ${{ !steps.set_outputs.outputs.trigger }} run: | sleep 300 exit 1 # fail out if the cancellation above somehow failed. - name: Cancel previous runs on the same PR - if: steps.set_outputs.outputs.trigger == 'label_trigger' + if: steps.set_outputs.outputs.trigger == 'label_trigger' uses: styfle/cancel-workflow-action@0.8.0 with: access_token: ${{ github.token }} @@ -161,14 +161,14 @@ jobs: command: pip install -r scripts/gha/requirements.txt - id: matrix_config run: | - if [[ "${{ steps.set_outputs.outputs.requested_tests }}" == "expanded" ]]; then + if [[ "${{ steps.set_outputs.outputs.requested_tests }}" == "expanded" ]]; then TEST_MATRIX_PARAM=-e=1 echo "::warning ::Running on the expanded matrix" - elif [[ "${{ steps.set_outputs.outputs.requested_tests }}" == "minimal" ]]; then + elif [[ "${{ steps.set_outputs.outputs.requested_tests }}" == "minimal" ]]; then TEST_MATRIX_PARAM=-m=1 echo "::warning ::Running on the minimal matrix" - elif [[ "${{ steps.set_outputs.outputs.requested_tests }}" == "auto" ]]; then - # auto-diff only apply when running in a PR. + elif [[ "${{ steps.set_outputs.outputs.requested_tests }}" == "auto" ]]; then + # auto-diff only apply when running in a PR. # diff against the PR's base. "git merge-base main branch_name" will give the common ancestor of both branches. MERGE_BASE=$(git merge-base origin/${{github.event.pull_request.head.ref}} origin/${{github.event.pull_request.base.ref}} || true) # If origin/ is no longer valid, then just run all tests. @@ -184,7 +184,7 @@ jobs: if [[ "${{ github.event.schedule }}" == "0 9 * * *" ]]; then # at 1am PST / 2am PDT. Running integration tests and generate test report for all testapps except firestore apis="admob,analytics,auth,database,dynamic_links,functions,installations,messaging,remote_config,storage" - elif [[ "${{ github.event.schedule }}" == "0 10 * * *" ]]; then + elif [[ "${{ github.event.schedule }}" == "0 10 * * *" ]]; then # at 2am PST / 3am PDT. Running integration tests for firestore and generate test report apis="firestore" else @@ -253,7 +253,7 @@ jobs: # Until we support building openssl from source, we can't link to system openssl when cross-compiling. # Builds all happen on x64 machines, so arm64 and x86 are technically cross-compiling. - ssl_variant: openssl - arch: arm64 + arch: arm64 - ssl_variant: openssl arch: x86 steps: @@ -350,7 +350,7 @@ jobs: run_id: ${{ github.event.inputs.test_packaged_sdk }} - name: Build integration tests shell: bash - env: + env: CCACHE_DIR: ${{ github.workspace }}/ccache_dir run: | declare -a additional_flags @@ -878,7 +878,7 @@ jobs: if: ${{ !cancelled() }} shell: bash run: | - # If testapps do not exist, then it's a build error not test error. + # If testapps do not exist, then it's a build error not test error. if [ -d "testapps/testapps-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}" && ! -f testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}.log.json ]; then mkdir -p testapps && echo "__SUMMARY_MISSING__" > testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}.log.json fi @@ -1123,7 +1123,7 @@ jobs: if: ${{ !cancelled() }} shell: bash run: | - # If testapps do not exist, then it's a build error not test error. + # If testapps do not exist, then it's a build error not test error. if [ -d "testapps/testapps-android-${{ matrix.build_os }}" && ! -f "testapps/test-results-android-${{ matrix.build_os }}-${{ matrix.android_device }}.log.json" ]; then mkdir -p testapps && echo "__SUMMARY_MISSING__" > "testapps/test-results-android-${{ matrix.build_os }}-${{ matrix.android_device }}.log.json" fi @@ -1215,7 +1215,7 @@ jobs: uses: google-github-actions/setup-gcloud@v0 - name: Run iOS integration tests on Real Device via FTL # max 3 retry and 10m timeout for each testapp, plus other steps - timeout-minutes: 60 + timeout-minutes: 60 if: steps.get-device-type.outputs.device_type == 'real' run: | python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}" @@ -1228,7 +1228,7 @@ jobs: if: ${{ !cancelled() }} shell: bash run: | - # If testapps do not exist, then it's a build error not test error. + # If testapps do not exist, then it's a build error not test error. if [ -d "testapps/testapps-ios-${{ matrix.build_os }}" && ! -f "testapps/test-results-ios-${{ matrix.build_os }}-${{ matrix.ios_device }}.log.json" ]; then mkdir -p testapps && echo "__SUMMARY_MISSING__" > "testapps/test-results-ios-${{ matrix.build_os }}-${{ matrix.ios_device }}.log.json" fi @@ -1311,7 +1311,7 @@ jobs: if: ${{ !cancelled() }} shell: bash run: | - # If testapps do not exist, then it's a build error not test error. + # If testapps do not exist, then it's a build error not test error. if [ -d "testapps/testapps-tvos-${{ matrix.build_os }}" && ! -f "testapps/test-results-tvos-${{ matrix.build_os }}-${{ matrix.tvos_device }}.log.json" ]; then mkdir -p testapps && echo "__SUMMARY_MISSING__" > "testapps/test-results-tvos-${{ matrix.build_os }}-${{ matrix.tvos_device }}.log.json" fi From 1ca44db8961bd436005a719bad006048c4909ddf Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 12:05:34 -0700 Subject: [PATCH 22/88] Comment --- .github/workflows/integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 71a0ab2c1..d21ff0af5 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -829,7 +829,7 @@ jobs: # Do not attempt to run for x86 on Mac. - os: macos-12 arch: x86 - # MacOS hosted runner cannot test arm64 code. + # MacOS GitHub-hosted runner cannot test arm64 code. - os: macos-12 arch: arm64 steps: From d3555e4689064e4a3406748d01d22d037e16e88a Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 12:08:47 -0700 Subject: [PATCH 23/88] Comments --- .github/workflows/integration_tests.yml | 29 ++++++++++++++++++++----- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index d21ff0af5..bb52bdbb2 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -240,7 +240,9 @@ jobs: os: ${{ fromJson(needs.check_and_prepare.outputs.matrix_os) }} arch: ${{ fromJson(needs.check_and_prepare.outputs.matrix_arch_combined) }} ssl_variant: ${{ fromJson(needs.check_and_prepare.outputs.matrix_ssl) }} - # Because matrix_arch_combined combines mac, linux, and windows, we need a few exclusions. + # Because matrix_arch_combined combines Mac, Linux, and Windows, we need + # a few exclusions. These are standard exclusions used in several + # places. exclude: # Do not attempt to build for arm64 on Windows or Linux. - os: windows-latest @@ -250,8 +252,9 @@ jobs: # Do not attempt to build for x86 on Mac. - os: macos-12 arch: x86 - # Until we support building openssl from source, we can't link to system openssl when cross-compiling. - # Builds all happen on x64 machines, so arm64 and x86 are technically cross-compiling. + # Until we support building openssl from source, we can't link to system + # openssl when cross-compiling. Builds all happen on x64 machines, so + # arm64 and x86 are technically cross-compiling. - ssl_variant: openssl arch: arm64 - ssl_variant: openssl @@ -819,17 +822,31 @@ jobs: os: ${{ fromJson(needs.check_and_prepare.outputs.matrix_os) }} arch: ${{ fromJson(needs.check_and_prepare.outputs.matrix_arch_combined) }} ssl_variant: ${{ fromJson(needs.check_and_prepare.outputs.matrix_ssl) }} - # Because matrix_arch_combined combines mac, linux, and windows, we need a few exclusions. + # Because matrix_arch_combined combines Mac, Linux, and Windows, we need + # a few exclusions. These are standard exclusions used in several + # places. exclude: # Do not attempt to run for arm64 on Windows or Linux. - os: windows-latest arch: arm64 - os: ubuntu-latest arch: arm64 - # Do not attempt to run for x86 on Mac. + # Do not attempt to run on x86 on Mac. - os: macos-12 arch: x86 - # MacOS GitHub-hosted runner cannot test arm64 code. + # Do not attempt to build for x86 on Mac. + - os: macos-12 + arch: x86 + # Until we support building openssl from source, we can't link to system + # openssl when cross-compiling. Builds all happen on x64 machines, so + # arm64 and x86 are technically cross-compiling. + - ssl_variant: openssl + arch: arm64 + - ssl_variant: openssl + arch: x86 + # Custom for this matrix: MacOS GitHub-hosted runner cannot test arm64 + # code. Exclude that scenario from running here; it will run in + # test_desktop_custom_runners. - os: macos-12 arch: arm64 steps: From 01707e3b48be522f5fc8dc5347cef466817f7c2f Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 12:08:47 -0700 Subject: [PATCH 24/88] Don't try to run openssl build on x86/arm64. Update comments. --- .github/workflows/integration_tests.yml | 29 ++++++++++++++++++++----- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index d21ff0af5..bb52bdbb2 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -240,7 +240,9 @@ jobs: os: ${{ fromJson(needs.check_and_prepare.outputs.matrix_os) }} arch: ${{ fromJson(needs.check_and_prepare.outputs.matrix_arch_combined) }} ssl_variant: ${{ fromJson(needs.check_and_prepare.outputs.matrix_ssl) }} - # Because matrix_arch_combined combines mac, linux, and windows, we need a few exclusions. + # Because matrix_arch_combined combines Mac, Linux, and Windows, we need + # a few exclusions. These are standard exclusions used in several + # places. exclude: # Do not attempt to build for arm64 on Windows or Linux. - os: windows-latest @@ -250,8 +252,9 @@ jobs: # Do not attempt to build for x86 on Mac. - os: macos-12 arch: x86 - # Until we support building openssl from source, we can't link to system openssl when cross-compiling. - # Builds all happen on x64 machines, so arm64 and x86 are technically cross-compiling. + # Until we support building openssl from source, we can't link to system + # openssl when cross-compiling. Builds all happen on x64 machines, so + # arm64 and x86 are technically cross-compiling. - ssl_variant: openssl arch: arm64 - ssl_variant: openssl @@ -819,17 +822,31 @@ jobs: os: ${{ fromJson(needs.check_and_prepare.outputs.matrix_os) }} arch: ${{ fromJson(needs.check_and_prepare.outputs.matrix_arch_combined) }} ssl_variant: ${{ fromJson(needs.check_and_prepare.outputs.matrix_ssl) }} - # Because matrix_arch_combined combines mac, linux, and windows, we need a few exclusions. + # Because matrix_arch_combined combines Mac, Linux, and Windows, we need + # a few exclusions. These are standard exclusions used in several + # places. exclude: # Do not attempt to run for arm64 on Windows or Linux. - os: windows-latest arch: arm64 - os: ubuntu-latest arch: arm64 - # Do not attempt to run for x86 on Mac. + # Do not attempt to run on x86 on Mac. - os: macos-12 arch: x86 - # MacOS GitHub-hosted runner cannot test arm64 code. + # Do not attempt to build for x86 on Mac. + - os: macos-12 + arch: x86 + # Until we support building openssl from source, we can't link to system + # openssl when cross-compiling. Builds all happen on x64 machines, so + # arm64 and x86 are technically cross-compiling. + - ssl_variant: openssl + arch: arm64 + - ssl_variant: openssl + arch: x86 + # Custom for this matrix: MacOS GitHub-hosted runner cannot test arm64 + # code. Exclude that scenario from running here; it will run in + # test_desktop_custom_runners. - os: macos-12 arch: arm64 steps: From a57b550dbd287ab7b99f8be31e95c9f2c44d1481 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 12:56:59 -0700 Subject: [PATCH 25/88] Pass in -j to cmake --build when building desktop testapps. This should speed up the desktop testapp build. --- scripts/gha/build_testapps.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/gha/build_testapps.py b/scripts/gha/build_testapps.py index 6a3d71785..043e55f48 100644 --- a/scripts/gha/build_testapps.py +++ b/scripts/gha/build_testapps.py @@ -177,6 +177,10 @@ "arch", "x64", "(Desktop only) Which architecture to build: x64 (all) or arm64 (Mac only).") +flags.DEFINE_integer( + "jobs", 3, + "(Desktop only) If > 0, pass in -j to CMake to parallelize build") + flags.DEFINE_multi_string( "cmake_flag", None, "Pass an additional flag to the CMake configure step." @@ -458,7 +462,8 @@ def _build_desktop(sdk_dir, cmake_flags): cmake_configure_cmd += ["-DCMAKE_OSX_ARCHITECTURES=%s" % ("arm64" if FLAGS.arch == "arm64" else "x86_64")] _run(cmake_configure_cmd + cmake_flags) - _run(["cmake", "--build", ".", "--config", "Debug"]) + _run(["cmake", "--build", ".", "--config", "Debug"] + + ["-j", "%s" % FLAGS.jobs] if FLAGS.jobs > 0 else []) def _get_desktop_compiler_flags(compiler, compiler_table): From 6f3f5823f8ab64b4afa2097fa7fa82e0ae48a30e Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 13:12:57 -0700 Subject: [PATCH 26/88] Change the default value for FLAGS.jobs to the # of cpu cores (max 4) --- scripts/gha/build_testapps.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/scripts/gha/build_testapps.py b/scripts/gha/build_testapps.py index 043e55f48..e20326bad 100644 --- a/scripts/gha/build_testapps.py +++ b/scripts/gha/build_testapps.py @@ -177,9 +177,19 @@ "arch", "x64", "(Desktop only) Which architecture to build: x64 (all) or arm64 (Mac only).") +# Get the number of CPUs for the default value of FLAGS.jobs +CPU_COUNT = os.cpu_count(); +# If CPU count couldn't be determined, default to 2. +DEFAULT_CPU_COUNT = 2 +if CPU_COUNT is None: CPU_COUNT = DEFAULT_CPU_COUNT +# Cap at 4 CPUs. +MAX_CPU_COUNT = 4 +if CPU_COUNT > MAX_CPU_COUNT: CPU_COUNT = MAX_CPU_COUNT + flags.DEFINE_integer( - "jobs", 3, - "(Desktop only) If > 0, pass in -j to CMake to parallelize build") + "jobs", CPU_COUNT, + "(Desktop only) If > 0, pass in -j to make CMake parallelize the" + " build. Defaults to the system's CPU count (max %s)." % MAX_CPU_COUNT) flags.DEFINE_multi_string( "cmake_flag", None, @@ -463,7 +473,7 @@ def _build_desktop(sdk_dir, cmake_flags): ("arm64" if FLAGS.arch == "arm64" else "x86_64")] _run(cmake_configure_cmd + cmake_flags) _run(["cmake", "--build", ".", "--config", "Debug"] + - ["-j", "%s" % FLAGS.jobs] if FLAGS.jobs > 0 else []) + ["-j", str(FLAGS.jobs)] if FLAGS.jobs > 0 else []) def _get_desktop_compiler_flags(compiler, compiler_table): From 805bbd9d1af259b358c4eea3e3cd664771b3b764 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 13:17:10 -0700 Subject: [PATCH 27/88] Actually build testapps with the correct arch --- .github/workflows/integration_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index bb52bdbb2..4a308111c 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -378,6 +378,7 @@ jobs: --noadd_timestamp \ --short_output_paths \ --gha_build \ + --arch ${{ matrix.arch }} \ ${additional_flags[*]} - name: Prepare results summary artifact if: ${{ !cancelled() }} From 7ba1c631f333a40d501b744e17da1d34873ee768 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 13:52:35 -0700 Subject: [PATCH 28/88] When building for Windows, use Win32 rather than x86 for -A. --- scripts/gha/build_testapps.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/gha/build_testapps.py b/scripts/gha/build_testapps.py index 8b6f5b5b3..b609f7132 100644 --- a/scripts/gha/build_testapps.py +++ b/scripts/gha/build_testapps.py @@ -460,7 +460,8 @@ def _build_desktop(sdk_dir, cmake_flags): cmake_configure_cmd = ["cmake", ".", "-DCMAKE_BUILD_TYPE=Debug", "-DFIREBASE_CPP_SDK_DIR=" + sdk_dir] if utils.is_windows_os(): - cmake_configure_cmd += ["-A", FLAGS.arch] + cmake_configure_cmd += ["-A", + "Win32" if FLAGS.arch == "x86" else FLAGS.arch] elif utils.is_mac_os(): # Ensure that correct Mac architecture is built. cmake_configure_cmd += ["-DCMAKE_OSX_ARCHITECTURES=%s" % From 5765dfc3b8537f4c377b4ff6693983f4bccdbae6 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 16:25:38 -0700 Subject: [PATCH 29/88] Install x86 prereqs on Linux before running tests. --- .github/workflows/integration_tests.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 4a308111c..35b8e5e45 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -854,6 +854,14 @@ jobs: - name: setup Xcode version (macos) if: runner.os == 'macOS' run: sudo xcode-select -s /Applications/Xcode_${{ fromJson(needs.check_and_prepare.outputs.xcode_version)[0] }}.app/Contents/Developer + - name: install x86 prereqs (linux) + if: runner.os == 'Linux' + run: | + set -x + sudo dpkg --add_architecture i386 + sudo apt update + # Linux tests require libsecret to be installed. + sudo apt install -V -y libsecret-1-0:i386 - uses: actions/checkout@v2 with: ref: ${{needs.check_and_prepare.outputs.github_ref}} From 62c57205d9f64907b2e972e24f911b6b3aa6d989 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 16:35:45 -0700 Subject: [PATCH 30/88] Rename custom runner job name --- .github/workflows/integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 35b8e5e45..d55a8cb2e 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -942,7 +942,7 @@ jobs: # Run tests that depend on custom (self-hosted) runners. # For now, this is only used for ARM Mac builds. test_desktop_custom_runners: - name: test-desktop-selfhosted-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} + name: test-desktop-custom-runner-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} needs: [check_and_prepare, build_desktop] runs-on: [self-hosted, firebase-cpp, '${{ matrix.runner_label }}' ] if: needs.check_and_prepare.outputs.apis != '' && !cancelled() From 1b42b15d125bf70ef5a66fe0acdf893ffdac4a74 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 17:10:47 -0700 Subject: [PATCH 31/88] Sync up test_desktop and test_desktop_custom_runners conditionals --- .github/workflows/integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index d55a8cb2e..8a31f0226 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -945,7 +945,7 @@ jobs: name: test-desktop-custom-runner-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} needs: [check_and_prepare, build_desktop] runs-on: [self-hosted, firebase-cpp, '${{ matrix.runner_label }}' ] - if: needs.check_and_prepare.outputs.apis != '' && !cancelled() + if: contains(needs.check_and_prepare.outputs.matrix_platform, 'Desktop') && needs.check_and_prepare.outputs.apis != '' && !cancelled() strategy: fail-fast: false matrix: From 5ff8a1f1ada1718ad8c9d587aaf134bb8b1ffd58 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 18:44:49 -0700 Subject: [PATCH 32/88] Fix Linux x86 tests --- .github/workflows/integration_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 8a31f0226..6e8a57954 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -855,10 +855,10 @@ jobs: if: runner.os == 'macOS' run: sudo xcode-select -s /Applications/Xcode_${{ fromJson(needs.check_and_prepare.outputs.xcode_version)[0] }}.app/Contents/Developer - name: install x86 prereqs (linux) - if: runner.os == 'Linux' + if: runner.os == 'Linux' && matrix.arch == 'x86' run: | set -x - sudo dpkg --add_architecture i386 + sudo dpkg --add-architecture i386 sudo apt update # Linux tests require libsecret to be installed. sudo apt install -V -y libsecret-1-0:i386 From 8e7038aa7cfbf99186a4b878630f235c8e1f2573 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Fri, 6 May 2022 18:47:28 -0700 Subject: [PATCH 33/88] Fix conditionals --- .github/workflows/integration_tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 6e8a57954..56a40cdfd 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -945,7 +945,7 @@ jobs: name: test-desktop-custom-runner-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} needs: [check_and_prepare, build_desktop] runs-on: [self-hosted, firebase-cpp, '${{ matrix.runner_label }}' ] - if: contains(needs.check_and_prepare.outputs.matrix_platform, 'Desktop') && needs.check_and_prepare.outputs.apis != '' && !cancelled() + if: contains(needs.check_and_prepare.outputs.matrix_platform, 'Desktop') && contains(needs.check_and_prepare.outputs.matrix_arch_combined, 'arm64') && needs.check_and_prepare.outputs.apis != '' && !cancelled() strategy: fail-fast: false matrix: @@ -959,15 +959,15 @@ jobs: runner_label: ${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }} - os: windows-latest runner_label: ${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }} - # For Mac, don't run x64 tests on arm64 runner. + # For Mac, don't run x86 or x64 tests. + - os: macos-12 + arch: x86 - os: macos-12 arch: x64 - runner_label: ${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }} # Until we support building openssl from source, we can't link to system # openssl when cross-compiling, so exclude openssl from mac arm64 testing. - os: macos-12 ssl_variant: openssl - runner_label: ${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }} steps: - name: Clean up previous run shell: bash From 30955d641920312de8b843b344bad91059062088 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sat, 7 May 2022 00:19:06 -0700 Subject: [PATCH 34/88] Install additional x86 linux package --- .github/workflows/integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 56a40cdfd..0e281aab5 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -861,7 +861,7 @@ jobs: sudo dpkg --add-architecture i386 sudo apt update # Linux tests require libsecret to be installed. - sudo apt install -V -y libsecret-1-0:i386 + sudo apt install -V -y libsecret-1-0:i386 libglib2.0-0:i386 - uses: actions/checkout@v2 with: ref: ${{needs.check_and_prepare.outputs.github_ref}} From 7da83cb7b0f71c0178bbe2b7c677fa8cb98ace13 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sat, 7 May 2022 00:22:18 -0700 Subject: [PATCH 35/88] Update simulator min version --- scripts/gha/print_matrix_configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gha/print_matrix_configuration.py b/scripts/gha/print_matrix_configuration.py index 952f38e3d..72c06ec47 100644 --- a/scripts/gha/print_matrix_configuration.py +++ b/scripts/gha/print_matrix_configuration.py @@ -172,7 +172,7 @@ "ios_min": {"type": "real", "model":"iphone8", "version":"11.4"}, "ios_target": {"type": "real", "model":"iphone8plus", "version":"12.0"}, "ios_latest": {"type": "real", "model":"iphone11pro", "version":"14.7"}, - "simulator_min": {"type": "virtual", "name":"iPhone 6", "version":"12.4"}, + "simulator_min": {"type": "virtual", "name":"iPhone 6", "version":"13.7"}, "simulator_target": {"type": "virtual", "name":"iPhone 8", "version":"14.5"}, "simulator_latest": {"type": "virtual", "name":"iPhone 11", "version":"15.2"}, "tvos_simulator": {"type": "virtual", "name":"Apple TV", "version":"14.3"}, From 9f57e5abf09e99588ece70731b75009bb39e557f Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sat, 7 May 2022 00:29:01 -0700 Subject: [PATCH 36/88] Flaky test handling for .info/connected on Android --- database/integration_test/src/integration_test.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/database/integration_test/src/integration_test.cc b/database/integration_test/src/integration_test.cc index ae4e7c68e..139f63a48 100644 --- a/database/integration_test/src/integration_test.cc +++ b/database/integration_test/src/integration_test.cc @@ -1275,11 +1275,11 @@ TEST_F(FirebaseDatabaseTest, TestInvalidatingReferencesWhenDeletingApp) { TEST_F(FirebaseDatabaseTest, TestInfoConnected) { SignIn(); - // The entire test can be a bit flaky on iOS, as the iOS SDK's - // .info/connected is not quite perfect. -#if defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE + // The entire test can be a bit flaky on mobile, as the iOS and + // Android SDKs' .info/connected is not quite perfect. +#if defined(ANDROID) || (defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE) FLAKY_TEST_SECTION_BEGIN(); -#endif // defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE +#endif // defined(ANDROID) || (defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE) firebase::database::DatabaseReference ref = CreateWorkingPath(); // Force getting a value so that we are connected to the database. @@ -1316,9 +1316,9 @@ TEST_F(FirebaseDatabaseTest, TestInfoConnected) { EXPECT_EQ(reconnected.result()->value(), true); } -#if defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE +#if defined(ANDROID) || (defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE) FLAKY_TEST_SECTION_END(); -#endif // defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE +#endif // defined(ANDROID) || (defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE) } TEST_F(FirebaseDatabaseTest, TestGetReferenceWillNullArgument) { From b33771ae8d610f9384b4857cd07c6b85db09f452 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sat, 7 May 2022 00:31:33 -0700 Subject: [PATCH 37/88] Fix matrix for custom runner job --- .github/workflows/integration_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 0e281aab5..4fd63be39 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -944,14 +944,14 @@ jobs: test_desktop_custom_runners: name: test-desktop-custom-runner-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} needs: [check_and_prepare, build_desktop] - runs-on: [self-hosted, firebase-cpp, '${{ matrix.runner_label }}' ] + runs-on: [self-hosted, firebase-cpp, '${{ matrix.runner_label }}'] if: contains(needs.check_and_prepare.outputs.matrix_platform, 'Desktop') && contains(needs.check_and_prepare.outputs.matrix_arch_combined, 'arm64') && needs.check_and_prepare.outputs.apis != '' && !cancelled() strategy: fail-fast: false matrix: ssl_variant: ${{ fromJson(needs.check_and_prepare.outputs.matrix_ssl) }} os: ${{ fromJson(needs.check_and_prepare.outputs.matrix_os) }} - arch: ${{ fromJson(needs.check_and_prepare.outputs.matrix_combined) }} + arch: ${{ fromJson(needs.check_and_prepare.outputs.matrix_arch_combined) }} runner_label: ['${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }}'] exclude: # Mac self-hosted runner shouldn't run Linux or Windows tests. From 22a66f193505ea6be5ba33ff7f85b82cdc9c9ffa Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sat, 7 May 2022 00:34:21 -0700 Subject: [PATCH 38/88] Fix merge --- scripts/gha/build_testapps.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/scripts/gha/build_testapps.py b/scripts/gha/build_testapps.py index 92e3ec5b4..e847a865e 100644 --- a/scripts/gha/build_testapps.py +++ b/scripts/gha/build_testapps.py @@ -187,20 +187,6 @@ MAX_CPU_COUNT = 4 if CPU_COUNT > MAX_CPU_COUNT: CPU_COUNT = MAX_CPU_COUNT -flags.DEFINE_integer( - "jobs", CPU_COUNT, - "(Desktop only) If > 0, pass in -j to make CMake parallelize the" - " build. Defaults to the system's CPU count (max %s)." % MAX_CPU_COUNT) - -# Get the number of CPUs for the default value of FLAGS.jobs -CPU_COUNT = os.cpu_count(); -# If CPU count couldn't be determined, default to 2. -DEFAULT_CPU_COUNT = 2 -if CPU_COUNT is None: CPU_COUNT = DEFAULT_CPU_COUNT -# Cap at 4 CPUs. -MAX_CPU_COUNT = 4 -if CPU_COUNT > MAX_CPU_COUNT: CPU_COUNT = MAX_CPU_COUNT - flags.DEFINE_integer( "jobs", CPU_COUNT, "(Desktop only) If > 0, pass in -j to make CMake parallelize the" From 1d832edc78c49ff77b86bf5b5323fa98ecd237d9 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sat, 7 May 2022 00:22:18 -0700 Subject: [PATCH 39/88] Update simulator min version --- scripts/gha/print_matrix_configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gha/print_matrix_configuration.py b/scripts/gha/print_matrix_configuration.py index c1bfa09a1..a80e68266 100644 --- a/scripts/gha/print_matrix_configuration.py +++ b/scripts/gha/print_matrix_configuration.py @@ -170,7 +170,7 @@ "ios_min": {"type": "real", "model":"iphone8", "version":"11.4"}, "ios_target": {"type": "real", "model":"iphone8plus", "version":"12.0"}, "ios_latest": {"type": "real", "model":"iphone11pro", "version":"14.7"}, - "simulator_min": {"type": "virtual", "name":"iPhone 6", "version":"12.4"}, + "simulator_min": {"type": "virtual", "name":"iPhone 6", "version":"13.7"}, "simulator_target": {"type": "virtual", "name":"iPhone 8", "version":"14.5"}, "simulator_latest": {"type": "virtual", "name":"iPhone 11", "version":"15.2"}, "tvos_simulator": {"type": "virtual", "name":"Apple TV", "version":"14.3"}, From d3830b9fe4ad266034c36ebbbeb389d37bfdc666 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sat, 7 May 2022 00:43:16 -0700 Subject: [PATCH 40/88] Enable openssl x86 builds --- .github/workflows/integration_tests.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 4fd63be39..8726caca5 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -253,12 +253,14 @@ jobs: - os: macos-12 arch: x86 # Until we support building openssl from source, we can't link to system - # openssl when cross-compiling. Builds all happen on x64 machines, so - # arm64 and x86 are technically cross-compiling. - - ssl_variant: openssl - arch: arm64 - - ssl_variant: openssl + # openssl when cross-compiling, except on Linux. Builds all happen on + # x64 machines, so arm64 and x86 are technically cross-compiling. + - os: windows-latest + ssl_variant: openssl arch: x86 + - os: macos-12 + ssl_variant: openssl + arch: arm64 steps: - name: setup Xcode version (macos) if: runner.os == 'macOS' @@ -860,8 +862,12 @@ jobs: set -x sudo dpkg --add-architecture i386 sudo apt update + additional_packages= # Linux tests require libsecret to be installed. - sudo apt install -V -y libsecret-1-0:i386 libglib2.0-0:i386 + if [[ '${{ matrix.ssl_variant }}' == 'openssl' ]]; then + additional_packages=libssl1.1:i386 + fi + sudo apt install -V -y libsecret-1-0:i386 libglib2.0-0:i386 ${additional_packages} - uses: actions/checkout@v2 with: ref: ${{needs.check_and_prepare.outputs.github_ref}} From f6e3096b6a84b411babdabb85f90b19cf9ab055b Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sat, 7 May 2022 00:52:13 -0700 Subject: [PATCH 41/88] Test openssl on linux x86 --- .github/workflows/integration_tests.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 8726caca5..20e730786 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -841,12 +841,14 @@ jobs: - os: macos-12 arch: x86 # Until we support building openssl from source, we can't link to system - # openssl when cross-compiling. Builds all happen on x64 machines, so - # arm64 and x86 are technically cross-compiling. - - ssl_variant: openssl - arch: arm64 - - ssl_variant: openssl + # openssl when cross-compiling, except on Linux. Builds all happen on + # x64 machines, so arm64 and x86 are technically cross-compiling. + - os: windows-latest + ssl_variant: openssl arch: x86 + - os: macos-12 + ssl_variant: openssl + arch: arm64 # Custom for this matrix: MacOS GitHub-hosted runner cannot test arm64 # code. Exclude that scenario from running here; it will run in # test_desktop_custom_runners. From d763746f99d7a0d577c4dae06249cd01ecb217ce Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sat, 7 May 2022 00:58:40 -0700 Subject: [PATCH 42/88] Clean up comments --- .github/workflows/integration_tests.yml | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 20e730786..5fc549d46 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -244,17 +244,18 @@ jobs: # a few exclusions. These are standard exclusions used in several # places. exclude: - # Do not attempt to build for arm64 on Windows or Linux. + # Do not attempt to use arm64 on Windows or Linux. - os: windows-latest arch: arm64 - os: ubuntu-latest arch: arm64 - # Do not attempt to build for x86 on Mac. + # Do not attempt to use x86 on Mac. - os: macos-12 arch: x86 - # Until we support building openssl from source, we can't link to system - # openssl when cross-compiling, except on Linux. Builds all happen on - # x64 machines, so arm64 and x86 are technically cross-compiling. + # Until we support building openssl from source, we can't use the + # system's openssl when cross-compiling, except on Linux. Builds all + # happen on x64 machines, so arm64 and x86 are technically + # cross-compiling. - os: windows-latest ssl_variant: openssl arch: x86 @@ -829,20 +830,18 @@ jobs: # a few exclusions. These are standard exclusions used in several # places. exclude: - # Do not attempt to run for arm64 on Windows or Linux. + # Do not attempt to use arm64 on Windows or Linux. - os: windows-latest arch: arm64 - os: ubuntu-latest arch: arm64 - # Do not attempt to run on x86 on Mac. - - os: macos-12 - arch: x86 - # Do not attempt to build for x86 on Mac. + # Do not attempt to use x86 on Mac. - os: macos-12 arch: x86 - # Until we support building openssl from source, we can't link to system - # openssl when cross-compiling, except on Linux. Builds all happen on - # x64 machines, so arm64 and x86 are technically cross-compiling. + # Until we support building openssl from source, we can't use the + # system's openssl when cross-compiling, except on Linux. Builds all + # happen on x64 machines, so arm64 and x86 are technically + # cross-compiling. - os: windows-latest ssl_variant: openssl arch: x86 From 9c09ff3982549139974eb378e9ce4a60e8213c00 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sat, 7 May 2022 01:02:29 -0700 Subject: [PATCH 43/88] Clean up exclusions --- .github/workflows/integration_tests.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 5fc549d46..6c00b9a01 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -961,19 +961,22 @@ jobs: arch: ${{ fromJson(needs.check_and_prepare.outputs.matrix_arch_combined) }} runner_label: ['${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }}'] exclude: - # Mac self-hosted runner shouldn't run Linux or Windows tests. + # Don't run Linux or Windows tests on the Mac arm64 runner. - os: ubuntu-latest runner_label: ${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }} - os: windows-latest runner_label: ${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }} - # For Mac, don't run x86 or x64 tests. + # Don't run x86 tests on any Mac runner. - os: macos-12 arch: x86 + # Don't run x64 tests on the Mac arm64 runner. - os: macos-12 arch: x64 + runner_label: ${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }} # Until we support building openssl from source, we can't link to system # openssl when cross-compiling, so exclude openssl from mac arm64 testing. - os: macos-12 + arch: arm64 ssl_variant: openssl steps: - name: Clean up previous run From 111db53c9eda38df178ab641fcc2a124329375b1 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sat, 7 May 2022 08:07:52 -0700 Subject: [PATCH 44/88] Add stdin option to restore_secrets for secure passphrase usage on runner. --- scripts/gha/restore_secrets.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/gha/restore_secrets.py b/scripts/gha/restore_secrets.py index 4a7305154..977594769 100644 --- a/scripts/gha/restore_secrets.py +++ b/scripts/gha/restore_secrets.py @@ -22,7 +22,7 @@ --passphrase: Passphrase to decrypt the files. This option is insecure on a multi-user machine; use the --passphrase_file option instead. --passphrase_file: Specify a file to read the passphrase from (only reads the - first line). + first line). Use "-" (without quotes) for stdin. --repo_dir: Path to C++ SDK Github repository. Defaults to current directory. This script will perform the following: @@ -48,7 +48,8 @@ flags.DEFINE_string("repo_dir", os.getcwd(), "Path to C++ SDK Github repo.") flags.DEFINE_string("passphrase", None, "The passphrase itself.") -flags.DEFINE_string("passphrase_file", None, "Path to file with passphrase.") +flags.DEFINE_string("passphrase_file", None, + "Path to file with passphrase. Use \"-\" (without quotes) for stdin.") flags.DEFINE_string("artifact", None, "Artifact Path, google-services.json will be placed here.") @@ -60,6 +61,8 @@ def main(argv): # The passphrase is sensitive, do not log. if FLAGS.passphrase: passphrase = FLAGS.passphrase + elif FLAGS.passphrase_file == "-": + passphrase = input() elif FLAGS.passphrase_file: with open(FLAGS.passphrase_file, "r") as f: passphrase = f.readline().strip() From d6a290ad0cb1da81ccef7f4eb6f0234396c64600 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sat, 7 May 2022 08:10:37 -0700 Subject: [PATCH 45/88] Add more linux x86 packages; also set up builder so we can get the error faster --- .github/workflows/integration_tests.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 6c00b9a01..6eec8b473 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -266,6 +266,18 @@ jobs: - name: setup Xcode version (macos) if: runner.os == 'macOS' run: sudo xcode-select -s /Applications/Xcode_${{ fromJson(needs.check_and_prepare.outputs.xcode_version)[0] }}.app/Contents/Developer + - name: install x86 prereqs (linux) + if: runner.os == 'Linux' && matrix.arch == 'x86' + run: | + set -x + sudo dpkg --add-architecture i386 + sudo apt update + additional_packages= + # Linux tests require libsecret to be installed. + if [[ '${{ matrix.ssl_variant }}' == 'openssl' ]]; then + additional_packages=libssl1.1:i386 + fi + sudo apt install -f -V -y libsecret-1-0:i386 libglib2.0-0:i386 libmount1:i386 libselinux1:i386 ${additional_packages} - uses: actions/checkout@v2 with: ref: ${{needs.check_and_prepare.outputs.github_ref}} @@ -868,7 +880,7 @@ jobs: if [[ '${{ matrix.ssl_variant }}' == 'openssl' ]]; then additional_packages=libssl1.1:i386 fi - sudo apt install -V -y libsecret-1-0:i386 libglib2.0-0:i386 ${additional_packages} + sudo apt install -f -V -y libsecret-1-0:i386 libglib2.0-0:i386 libmount1:i386 libselinux1:i386 ${additional_packages} - uses: actions/checkout@v2 with: ref: ${{needs.check_and_prepare.outputs.github_ref}} From 25be6498c2f0ae16cd3d7e972bd3727a41d68d40 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sat, 7 May 2022 08:18:17 -0700 Subject: [PATCH 46/88] x86 linux libs --- .github/workflows/integration_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 6eec8b473..bdacee460 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -277,7 +277,7 @@ jobs: if [[ '${{ matrix.ssl_variant }}' == 'openssl' ]]; then additional_packages=libssl1.1:i386 fi - sudo apt install -f -V -y libsecret-1-0:i386 libglib2.0-0:i386 libmount1:i386 libselinux1:i386 ${additional_packages} + sudo apt install -f -V -y libsecret-1-0:i386 libglib2.0-0:i386 libmount1:i386 libselinux1:i386 libpcre2-8-0:i386 ${additional_packages} - uses: actions/checkout@v2 with: ref: ${{needs.check_and_prepare.outputs.github_ref}} @@ -880,7 +880,7 @@ jobs: if [[ '${{ matrix.ssl_variant }}' == 'openssl' ]]; then additional_packages=libssl1.1:i386 fi - sudo apt install -f -V -y libsecret-1-0:i386 libglib2.0-0:i386 libmount1:i386 libselinux1:i386 ${additional_packages} + sudo apt install -f -V -y libsecret-1-0:i386 libglib2.0-0:i386 libmount1:i386 libselinux1:i386 libpcre2-8-0:i386 ${additional_packages} - uses: actions/checkout@v2 with: ref: ${{needs.check_and_prepare.outputs.github_ref}} From 7499d76a2ab1510831bd09d7c5c497b4cc2158fb Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sat, 7 May 2022 09:52:55 -0700 Subject: [PATCH 47/88] Move x86 support library installtion code to a standalone script. --- .github/workflows/integration_tests.yml | 15 +--- scripts/gha/build_desktop.py | 45 +---------- scripts/gha/install_x86_support_libraries.py | 85 ++++++++++++++++++++ 3 files changed, 90 insertions(+), 55 deletions(-) create mode 100644 scripts/gha/install_x86_support_libraries.py diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index bdacee460..b22326d11 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -266,22 +266,13 @@ jobs: - name: setup Xcode version (macos) if: runner.os == 'macOS' run: sudo xcode-select -s /Applications/Xcode_${{ fromJson(needs.check_and_prepare.outputs.xcode_version)[0] }}.app/Contents/Developer - - name: install x86 prereqs (linux) - if: runner.os == 'Linux' && matrix.arch == 'x86' - run: | - set -x - sudo dpkg --add-architecture i386 - sudo apt update - additional_packages= - # Linux tests require libsecret to be installed. - if [[ '${{ matrix.ssl_variant }}' == 'openssl' ]]; then - additional_packages=libssl1.1:i386 - fi - sudo apt install -f -V -y libsecret-1-0:i386 libglib2.0-0:i386 libmount1:i386 libselinux1:i386 libpcre2-8-0:i386 ${additional_packages} - uses: actions/checkout@v2 with: ref: ${{needs.check_and_prepare.outputs.github_ref}} submodules: true + - name: install x86 prereqs (linux) + if: runner.os == 'Linux' && matrix.arch == 'x86' + run: python scripts/gha/install_x86_support_libraries.py --gha_build - name: Store git credentials for all git commands # Forces all git commands to use authenticated https, to prevent throttling. shell: bash diff --git a/scripts/gha/build_desktop.py b/scripts/gha/build_desktop.py index 1978ff0ba..02c8928ee 100644 --- a/scripts/gha/build_desktop.py +++ b/scripts/gha/build_desktop.py @@ -40,6 +40,7 @@ import shutil import subprocess import sys +from install_x86_support_libraries import install def append_line_to_file(path, line): """Append the given line to the end of the file if it's not already in the file. @@ -57,48 +58,6 @@ def append_line_to_file(path, line): file.write("\n" + line + "\n") -def install_x86_support_libraries(gha_build=False): - """Install support libraries needed to build x86 on x86_64 hosts. - - Args: - gha_build: Pass in True if running on a GitHub runner; this will activate - workarounds that might be undesirable on a personal system (e.g. - downgrading Ubuntu packages). - """ - if utils.is_linux_os(): - packages = ['gcc-multilib', 'g++-multilib', 'libglib2.0-dev:i386', - 'libsecret-1-dev:i386', 'libpthread-stubs0-dev:i386', - 'libssl-dev:i386'] - if gha_build: - # Workaround for GitHub runners, which have an incompatibility between the - # 64-bit and 32-bit versions of the Ubuntu package libpcre2-8-0. Downgrade - # the installed 64-bit version of the library to get around this issue. - # This will presumably be fixed in a future Ubuntu update. (If you remove - # it, remove the workaround further down this function as well.) - packages = ['--allow-downgrades'] + packages + ['libpcre2-8-0=10.34-7'] - - # First check if these packages exist on the machine already - devnull = open(os.devnull, "w") - process = subprocess.run(["dpkg", "-s"] + packages, stdout=devnull, stderr=subprocess.STDOUT) - devnull.close() - if process.returncode != 0: - # This implies not all of the required packages are already installed on user's machine - # Install them. - utils.run_command(['dpkg', '--add-architecture', 'i386'], as_root=True, check=True) - utils.run_command(['apt', 'update'], as_root=True, check=True) - utils.run_command(['apt', 'install', '-V', '-y'] + packages, as_root=True, check=True) - - if gha_build: - # One more workaround: downgrading libpcre2-8-0 above may have uninstalled - # libsecret, which is required for the Linux build. Force it to be - # reinstalled, but do it as a separate command to ensure that held - # packages aren't modified. (Once the workaround above is removed, this can - # be removed as well.) - # Note: "-f" = "fix" - let apt do what it needs to do to fix dependencies. - utils.run_command(['apt', 'install', '-f', '-V', '-y', 'libsecret-1-dev'], - as_root=True, check=True) - - def _install_cpp_dependencies_with_vcpkg(arch, msvc_runtime_library, use_openssl=False): """Install packages with vcpkg. @@ -281,7 +240,7 @@ def main(): # To build x86 on x86_64 linux hosts, we also need x86 support libraries if args.arch == 'x86' and utils.is_linux_os(): - install_x86_support_libraries(args.gha_build) + install_x86_support_libraries.install(args.gha_build) # Install C++ dependencies using vcpkg if not args.disable_vcpkg: diff --git a/scripts/gha/install_x86_support_libraries.py b/scripts/gha/install_x86_support_libraries.py new file mode 100644 index 000000000..f930b7932 --- /dev/null +++ b/scripts/gha/install_x86_support_libraries.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python + +# Copyright 2020 Google +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Install Linux x86 support libraries. +""" + +import argparse +import os +import utils +import shutil +import subprocess +import sys + + +def install(gha_build=False): + """Install support libraries needed to build x86 on x86_64 hosts. + + Args: + gha_build: Pass in True if running on a GitHub runner; this will activate + workarounds that might be undesirable on a personal system (e.g. + downgrading Ubuntu packages). + """ + if utils.is_linux_os(): + packages = ['gcc-multilib', 'g++-multilib', 'libglib2.0-dev:i386', + 'libsecret-1-dev:i386', 'libpthread-stubs0-dev:i386', + 'libssl-dev:i386'] + if gha_build: + # Workaround for GitHub runners, which have an incompatibility between the + # 64-bit and 32-bit versions of the Ubuntu package libpcre2-8-0. Downgrade + # the installed 64-bit version of the library to get around this issue. + # This will presumably be fixed in a future Ubuntu update. (If you remove + # it, remove the workaround further down this function as well.) + packages = ['--allow-downgrades'] + packages + ['libpcre2-8-0=10.34-7'] + + # First check if these packages exist on the machine already + devnull = open(os.devnull, "w") + process = subprocess.run(["dpkg", "-s"] + packages, stdout=devnull, stderr=subprocess.STDOUT) + devnull.close() + if process.returncode != 0: + # This implies not all of the required packages are already installed on user's machine + # Install them. + utils.run_command(['dpkg', '--add-architecture', 'i386'], as_root=True, check=True) + utils.run_command(['apt', 'update'], as_root=True, check=True) + utils.run_command(['apt', 'install', '-V', '-y'] + packages, as_root=True, check=True) + + if gha_build: + # One more workaround: downgrading libpcre2-8-0 above may have uninstalled + # libsecret, which is required for the Linux build. Force it to be + # reinstalled, but do it as a separate command to ensure that held + # packages aren't modified. (Once the workaround above is removed, this can + # be removed as well.) + # Note: "-f" = "fix" - let apt do what it needs to do to fix dependencies. + utils.run_command(['apt', 'install', '-f', '-V', '-y', 'libsecret-1-dev'], + as_root=True, check=True) + + +def main(): + args = parse_cmdline_args() + if not utils.is_linux_os(): + print("This tool is for Linux only.") + exit(1) + install(args.gha_build) + + +def parse_cmdline_args(): + parser = argparse.ArgumentParser(description='Install Linux x86 support packages') + parser.add_argument('--gha_build', action='store_true', default=None, help='Set to true when building on GitHub. Changes some prerequisite installation behavior to GitHub-specific workarounds.') + args = parser.parse_args() + return args + +if __name__ == '__main__': + main() From 660a30bd047640bdf21791f2efd8021b5e61ca94 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sat, 7 May 2022 09:54:40 -0700 Subject: [PATCH 48/88] Fix installation of x86 support libs --- .github/workflows/integration_tests.yml | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index b22326d11..3d3ef3f55 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -270,9 +270,6 @@ jobs: with: ref: ${{needs.check_and_prepare.outputs.github_ref}} submodules: true - - name: install x86 prereqs (linux) - if: runner.os == 'Linux' && matrix.arch == 'x86' - run: python scripts/gha/install_x86_support_libraries.py --gha_build - name: Store git credentials for all git commands # Forces all git commands to use authenticated https, to prevent throttling. shell: bash @@ -305,6 +302,9 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ env.pythonVersion }} + - name: install x86 support libraries (linux) + if: runner.os == 'Linux' && matrix.arch == 'x86' + run: python scripts/gha/install_x86_support_libraries.py --gha_build - name: Update homebrew (avoid bintray errors) uses: nick-invision/retry@v2 if: startsWith(matrix.os, 'macos') @@ -860,21 +860,9 @@ jobs: - name: setup Xcode version (macos) if: runner.os == 'macOS' run: sudo xcode-select -s /Applications/Xcode_${{ fromJson(needs.check_and_prepare.outputs.xcode_version)[0] }}.app/Contents/Developer - - name: install x86 prereqs (linux) + - name: install x86 support libraries (linux) if: runner.os == 'Linux' && matrix.arch == 'x86' - run: | - set -x - sudo dpkg --add-architecture i386 - sudo apt update - additional_packages= - # Linux tests require libsecret to be installed. - if [[ '${{ matrix.ssl_variant }}' == 'openssl' ]]; then - additional_packages=libssl1.1:i386 - fi - sudo apt install -f -V -y libsecret-1-0:i386 libglib2.0-0:i386 libmount1:i386 libselinux1:i386 libpcre2-8-0:i386 ${additional_packages} - - uses: actions/checkout@v2 - with: - ref: ${{needs.check_and_prepare.outputs.github_ref}} + run: python scripts/gha/install_x86_support_libraries.py --gha_build - name: Download Desktop integration tests artifact uses: actions/download-artifact@v3 with: @@ -884,6 +872,9 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ env.pythonVersion }} + - uses: actions/checkout@v2 + with: + ref: ${{needs.check_and_prepare.outputs.github_ref}} - name: Set up Node (12) uses: actions/setup-node@v2 with: From bdc1a40a1da54c3cc7fe69cbc74731aa65ffdeb9 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sat, 7 May 2022 10:23:07 -0700 Subject: [PATCH 49/88] Fix filename --- .github/workflows/integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 3d3ef3f55..44a9be3a5 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -1070,7 +1070,7 @@ jobs: if: ${{ !cancelled() }} shell: bash run: | - cat testapps/test-results-desktop-${{ matrix.runner_label }}-${{ matrix.ssl_variant }}.log + cat testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}.log if [[ "${{ job.status }}" != "success" ]]; then exit 1 fi From 96558d937112eac703e30c7c62ded0584592c6a2 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sat, 7 May 2022 10:56:49 -0700 Subject: [PATCH 50/88] Move install_x86_support_libraries to utils. --- scripts/gha/build_desktop.py | 3 +- scripts/gha/install_x86_support_libraries.py | 85 -------------------- scripts/gha/utils.py | 42 ++++++++++ 3 files changed, 43 insertions(+), 87 deletions(-) delete mode 100644 scripts/gha/install_x86_support_libraries.py diff --git a/scripts/gha/build_desktop.py b/scripts/gha/build_desktop.py index 02c8928ee..b8c1d9f70 100644 --- a/scripts/gha/build_desktop.py +++ b/scripts/gha/build_desktop.py @@ -40,7 +40,6 @@ import shutil import subprocess import sys -from install_x86_support_libraries import install def append_line_to_file(path, line): """Append the given line to the end of the file if it's not already in the file. @@ -240,7 +239,7 @@ def main(): # To build x86 on x86_64 linux hosts, we also need x86 support libraries if args.arch == 'x86' and utils.is_linux_os(): - install_x86_support_libraries.install(args.gha_build) + utils.install_x86_support_libraries(args.gha_build) # Install C++ dependencies using vcpkg if not args.disable_vcpkg: diff --git a/scripts/gha/install_x86_support_libraries.py b/scripts/gha/install_x86_support_libraries.py deleted file mode 100644 index f930b7932..000000000 --- a/scripts/gha/install_x86_support_libraries.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2020 Google -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Install Linux x86 support libraries. -""" - -import argparse -import os -import utils -import shutil -import subprocess -import sys - - -def install(gha_build=False): - """Install support libraries needed to build x86 on x86_64 hosts. - - Args: - gha_build: Pass in True if running on a GitHub runner; this will activate - workarounds that might be undesirable on a personal system (e.g. - downgrading Ubuntu packages). - """ - if utils.is_linux_os(): - packages = ['gcc-multilib', 'g++-multilib', 'libglib2.0-dev:i386', - 'libsecret-1-dev:i386', 'libpthread-stubs0-dev:i386', - 'libssl-dev:i386'] - if gha_build: - # Workaround for GitHub runners, which have an incompatibility between the - # 64-bit and 32-bit versions of the Ubuntu package libpcre2-8-0. Downgrade - # the installed 64-bit version of the library to get around this issue. - # This will presumably be fixed in a future Ubuntu update. (If you remove - # it, remove the workaround further down this function as well.) - packages = ['--allow-downgrades'] + packages + ['libpcre2-8-0=10.34-7'] - - # First check if these packages exist on the machine already - devnull = open(os.devnull, "w") - process = subprocess.run(["dpkg", "-s"] + packages, stdout=devnull, stderr=subprocess.STDOUT) - devnull.close() - if process.returncode != 0: - # This implies not all of the required packages are already installed on user's machine - # Install them. - utils.run_command(['dpkg', '--add-architecture', 'i386'], as_root=True, check=True) - utils.run_command(['apt', 'update'], as_root=True, check=True) - utils.run_command(['apt', 'install', '-V', '-y'] + packages, as_root=True, check=True) - - if gha_build: - # One more workaround: downgrading libpcre2-8-0 above may have uninstalled - # libsecret, which is required for the Linux build. Force it to be - # reinstalled, but do it as a separate command to ensure that held - # packages aren't modified. (Once the workaround above is removed, this can - # be removed as well.) - # Note: "-f" = "fix" - let apt do what it needs to do to fix dependencies. - utils.run_command(['apt', 'install', '-f', '-V', '-y', 'libsecret-1-dev'], - as_root=True, check=True) - - -def main(): - args = parse_cmdline_args() - if not utils.is_linux_os(): - print("This tool is for Linux only.") - exit(1) - install(args.gha_build) - - -def parse_cmdline_args(): - parser = argparse.ArgumentParser(description='Install Linux x86 support packages') - parser.add_argument('--gha_build', action='store_true', default=None, help='Set to true when building on GitHub. Changes some prerequisite installation behavior to GitHub-specific workarounds.') - args = parser.parse_args() - return args - -if __name__ == '__main__': - main() diff --git a/scripts/gha/utils.py b/scripts/gha/utils.py index 9201fed01..7f964e414 100644 --- a/scripts/gha/utils.py +++ b/scripts/gha/utils.py @@ -218,3 +218,45 @@ def clean_vcpkg_temp_data(): for directory_to_remove in directories_to_remove: abspath = os.path.join(vcpkg_root_dir_path, directory_to_remove) delete_directory(abspath) + + +def install_x86_support_libraries(gha_build=False): + """Install support libraries needed to build x86 on x86_64 hosts. + + Args: + gha_build: Pass in True if running on a GitHub runner; this will activate + workarounds that might be undesirable on a personal system (e.g. + downgrading Ubuntu packages). + """ + if is_linux_os(): + packages = ['gcc-multilib', 'g++-multilib', 'libglib2.0-dev:i386', + 'libsecret-1-dev:i386', 'libpthread-stubs0-dev:i386', + 'libssl-dev:i386'] + if gha_build: + # Workaround for GitHub runners, which have an incompatibility between the + # 64-bit and 32-bit versions of the Ubuntu package libpcre2-8-0. Downgrade + # the installed 64-bit version of the library to get around this issue. + # This will presumably be fixed in a future Ubuntu update. (If you remove + # it, remove the workaround further down this function as well.) + packages = ['--allow-downgrades'] + packages + ['libpcre2-8-0=10.34-7'] + + # First check if these packages exist on the machine already + devnull = open(os.devnull, "w") + process = subprocess.run(["dpkg", "-s"] + packages, stdout=devnull, stderr=subprocess.STDOUT) + devnull.close() + if process.returncode != 0: + # This implies not all of the required packages are already installed on user's machine + # Install them. + run_command(['dpkg', '--add-architecture', 'i386'], as_root=True, check=True) + run_command(['apt', 'update'], as_root=True, check=True) + run_command(['apt', 'install', '-V', '-y'] + packages, as_root=True, check=True) + + if gha_build: + # One more workaround: downgrading libpcre2-8-0 above may have uninstalled + # libsecret, which is required for the Linux build. Force it to be + # reinstalled, but do it as a separate command to ensure that held + # packages aren't modified. (Once the workaround above is removed, this can + # be removed as well.) + # Note: "-f" = "fix" - let apt do what it needs to do to fix dependencies. + run_command(['apt', 'install', '-f', '-V', '-y', 'libsecret-1-dev'], + as_root=True, check=True) From 4c6c169a5b63c1abe2a5575477e6a260024ce7f3 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sat, 7 May 2022 11:12:43 -0700 Subject: [PATCH 51/88] Add generic --arch command for install_prereqs_desktop. --- .github/workflows/integration_tests.yml | 12 ++- scripts/gha/install_prereqs_desktop.py | 112 +++++++++++++----------- 2 files changed, 68 insertions(+), 56 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 44a9be3a5..cede28d4f 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -302,9 +302,6 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ env.pythonVersion }} - - name: install x86 support libraries (linux) - if: runner.os == 'Linux' && matrix.arch == 'x86' - run: python scripts/gha/install_x86_support_libraries.py --gha_build - name: Update homebrew (avoid bintray errors) uses: nick-invision/retry@v2 if: startsWith(matrix.os, 'macos') @@ -322,7 +319,7 @@ jobs: timeout_minutes: 2 max_attempts: 3 command: | - python scripts/gha/install_prereqs_desktop.py + python scripts/gha/install_prereqs_desktop.py --gha_build --arch '${{ matrix.arch }}' pip install -r scripts/gha/requirements.txt python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}" - name: Install OpenSSL (Windows) @@ -860,9 +857,6 @@ jobs: - name: setup Xcode version (macos) if: runner.os == 'macOS' run: sudo xcode-select -s /Applications/Xcode_${{ fromJson(needs.check_and_prepare.outputs.xcode_version)[0] }}.app/Contents/Developer - - name: install x86 support libraries (linux) - if: runner.os == 'Linux' && matrix.arch == 'x86' - run: python scripts/gha/install_x86_support_libraries.py --gha_build - name: Download Desktop integration tests artifact uses: actions/download-artifact@v3 with: @@ -875,6 +869,10 @@ jobs: - uses: actions/checkout@v2 with: ref: ${{needs.check_and_prepare.outputs.github_ref}} + - name: Install Desktop SDK & integration tests prerequisites + uses: nick-invision/retry@v2 + - name: install platform support libraries for running + run: python scripts/gha/install_prereqs_desktop.py --gha_build --runner_only --arch '${{ matrix.arch }}' - name: Set up Node (12) uses: actions/setup-node@v2 with: diff --git a/scripts/gha/install_prereqs_desktop.py b/scripts/gha/install_prereqs_desktop.py index a8511c493..c3d1b1403 100644 --- a/scripts/gha/install_prereqs_desktop.py +++ b/scripts/gha/install_prereqs_desktop.py @@ -29,62 +29,76 @@ """ +import argparse import utils def main(): - # Install protobuf on linux/mac if its not installed already - if not utils.is_command_installed('protoc'): - if utils.is_linux_os(): - # sudo apt install protobuf-compiler - utils.run_command(['apt', 'install', '-y','protobuf-compiler'], as_root=True) - elif utils.is_mac_os(): - # brew install protobuf - utils.run_command(['brew', 'install', 'protobuf']) - - # Install go on linux/mac if its not installed already - if not utils.is_command_installed('go'): - if utils.is_linux_os(): - # sudo apt install -y golang - utils.run_command(['apt', 'install', '-y','golang'], as_root=True) - elif utils.is_mac_os(): - # brew install protobuf - utils.run_command(['brew', 'install', 'go']) - - # Install openssl on linux/mac if its not installed already - if not utils.is_command_installed('go'): - if utils.is_linux_os(): - # sudo apt install -y openssl - utils.run_command(['apt', 'install', '-y','openssl'], as_root=True) - elif utils.is_mac_os(): - # brew install protobuf - utils.run_command(['brew', 'install', 'openssl']) + args = parse_cmdline_args() - # Install ccache on linux/mac if its not installed already - if not utils.is_command_installed('ccache'): - if utils.is_linux_os(): - # sudo apt install ccache - utils.run_command(['apt', 'install', '-y', 'ccache'], as_root=True) - elif utils.is_mac_os(): - # brew install ccache - utils.run_command(['brew', 'install', 'ccache']) + if not args.run_only: + # Install protobuf on linux/mac if its not installed already + if not utils.is_command_installed('protoc'): + if utils.is_linux_os(): + # sudo apt install protobuf-compiler + utils.run_command(['apt', 'install', '-y','protobuf-compiler'], as_root=True) + elif utils.is_mac_os(): + # brew install protobuf + utils.run_command(['brew', 'install', 'protobuf']) + + # Install go on linux/mac if its not installed already + if not utils.is_command_installed('go'): + if utils.is_linux_os(): + # sudo apt install -y golang + utils.run_command(['apt', 'install', '-y','golang'], as_root=True) + elif utils.is_mac_os(): + # brew install protobuf + utils.run_command(['brew', 'install', 'go']) + + # Install openssl on linux/mac if its not installed already + if not utils.is_command_installed('go'): + if utils.is_linux_os(): + # sudo apt install -y openssl + utils.run_command(['apt', 'install', '-y','openssl'], as_root=True) + elif utils.is_mac_os(): + # brew install protobuf + utils.run_command(['brew', 'install', 'openssl']) + + # Install ccache on linux/mac if its not installed already + if not utils.is_command_installed('ccache'): + if utils.is_linux_os(): + # sudo apt install ccache + utils.run_command(['apt', 'install', '-y', 'ccache'], as_root=True) + elif utils.is_mac_os(): + # brew install ccache + utils.run_command(['brew', 'install', 'ccache']) + + # Install clang-format on linux/mac if its not installed already + if not utils.is_command_installed('clang-format'): + if utils.is_linux_os(): + # sudo apt install clang-format + utils.run_command(['apt', 'install', '-y','clang-format'], as_root=True) + elif utils.is_mac_os(): + # brew install protobuf + utils.run_command(['brew', 'install', 'clang-format']) - # Install clang-format on linux/mac if its not installed already - if not utils.is_command_installed('clang-format'): - if utils.is_linux_os(): - # sudo apt install clang-format - utils.run_command(['apt', 'install', '-y','clang-format'], as_root=True) - elif utils.is_mac_os(): - # brew install protobuf - utils.run_command(['brew', 'install', 'clang-format']) + # Install required python dependencies. + # On Catalina, python2 in installed as default python. + # Example command: + # python3 -m pip install -r external/pip_requirements.txt --user + utils.run_command( + ['python3' if utils.is_command_installed('python3') else 'python', '-m', + 'pip', 'install', '-r', 'external/pip_requirements.txt', '--user'] ) - # Install required python dependencies. - # On Catalina, python2 in installed as default python. - # Example command: - # python3 -m pip install -r external/pip_requirements.txt --user - utils.run_command( - ['python3' if utils.is_command_installed('python3') else 'python', '-m', - 'pip', 'install', '-r', 'external/pip_requirements.txt', '--user'] ) + if args.arch == 'x86': + utils.install_x86_support_libraries(args.gha_build) +def parse_cmdline_args(): + parser = argparse.ArgumentParser(description='Install prerequisites for building cpp sdk') + parser.add_argument('--arch', default=None, help='Install support libraries to build a specific architecture (currently supported: x86)') + parser.add_argument('--running_only', action='store_true', help='Only install prerequisites for running, not for building') + parser.add_argument('--gha_build', action='store_true', default=None, help='Set to true when building on GitHub, for metric tracking purposes (also changes some prerequisite installation behavior).') + args = parser.parse_args() + return args if __name__ == '__main__': main() From cc4931236774ab7503011b45211b4178ef30e778 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sat, 7 May 2022 11:14:34 -0700 Subject: [PATCH 52/88] Fix parameter --- .github/workflows/integration_tests.yml | 2 +- scripts/gha/install_prereqs_desktop.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index cede28d4f..3720128ff 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -872,7 +872,7 @@ jobs: - name: Install Desktop SDK & integration tests prerequisites uses: nick-invision/retry@v2 - name: install platform support libraries for running - run: python scripts/gha/install_prereqs_desktop.py --gha_build --runner_only --arch '${{ matrix.arch }}' + run: python scripts/gha/install_prereqs_desktop.py --gha_build --running_only --arch '${{ matrix.arch }}' - name: Set up Node (12) uses: actions/setup-node@v2 with: diff --git a/scripts/gha/install_prereqs_desktop.py b/scripts/gha/install_prereqs_desktop.py index c3d1b1403..756ad7472 100644 --- a/scripts/gha/install_prereqs_desktop.py +++ b/scripts/gha/install_prereqs_desktop.py @@ -35,7 +35,7 @@ def main(): args = parse_cmdline_args() - if not args.run_only: + if not args.running_only: # Install protobuf on linux/mac if its not installed already if not utils.is_command_installed('protoc'): if utils.is_linux_os(): From 51b9cebb9211335196d51f5dc76c271db08507aa Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sat, 7 May 2022 15:00:49 -0700 Subject: [PATCH 53/88] Fix prereq step --- .github/workflows/integration_tests.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 3720128ff..4a07d52c1 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -869,10 +869,12 @@ jobs: - uses: actions/checkout@v2 with: ref: ${{needs.check_and_prepare.outputs.github_ref}} - - name: Install Desktop SDK & integration tests prerequisites - uses: nick-invision/retry@v2 - name: install platform support libraries for running - run: python scripts/gha/install_prereqs_desktop.py --gha_build --running_only --arch '${{ matrix.arch }}' + uses: nick-invision/retry@v2 + with: + timeout_minutes: 2 + max_attempts: 3 + command: python scripts/gha/install_prereqs_desktop.py --gha_build --running_only --arch '${{ matrix.arch }}' - name: Set up Node (12) uses: actions/setup-node@v2 with: From 2c9076a0b7a54e552be307b7351995dfe38c941e Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sat, 7 May 2022 15:07:07 -0700 Subject: [PATCH 54/88] Increase timeout --- .github/workflows/integration_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 4a07d52c1..edfb453ae 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -316,7 +316,7 @@ jobs: - name: Install Desktop SDK & integration tests prerequisites uses: nick-invision/retry@v2 with: - timeout_minutes: 2 + timeout_minutes: 15 max_attempts: 3 command: | python scripts/gha/install_prereqs_desktop.py --gha_build --arch '${{ matrix.arch }}' @@ -872,7 +872,7 @@ jobs: - name: install platform support libraries for running uses: nick-invision/retry@v2 with: - timeout_minutes: 2 + timeout_minutes: 15 max_attempts: 3 command: python scripts/gha/install_prereqs_desktop.py --gha_build --running_only --arch '${{ matrix.arch }}' - name: Set up Node (12) From da34918c474bbf5e32ad66381ec426f16fa29ec7 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sat, 7 May 2022 18:05:47 -0700 Subject: [PATCH 55/88] Move checkout earlier again --- .github/workflows/integration_tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index edfb453ae..f824ff67a 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -857,6 +857,9 @@ jobs: - name: setup Xcode version (macos) if: runner.os == 'macOS' run: sudo xcode-select -s /Applications/Xcode_${{ fromJson(needs.check_and_prepare.outputs.xcode_version)[0] }}.app/Contents/Developer + - uses: actions/checkout@v2 + with: + ref: ${{needs.check_and_prepare.outputs.github_ref}} - name: Download Desktop integration tests artifact uses: actions/download-artifact@v3 with: @@ -866,9 +869,6 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ env.pythonVersion }} - - uses: actions/checkout@v2 - with: - ref: ${{needs.check_and_prepare.outputs.github_ref}} - name: install platform support libraries for running uses: nick-invision/retry@v2 with: From f321f349c27d1058eebfcaf208b66edeaf1a419c Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sat, 7 May 2022 18:31:02 -0700 Subject: [PATCH 56/88] Clean up custom runner shell script. --- .github/workflows/integration_tests.yml | 26 +++++++++++++++++-------- scripts/gha/desktop_tester.py | 6 ++++-- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index f824ff67a..4be189285 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -1017,15 +1017,25 @@ jobs: shell: bash run: | if [[ '${{ matrix.runner_label }}' == '${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }}' ]]; then - # Mac ARM64 only: Use arch -arm64 prefix to ensure that the app is running on arm64. - cmd_prefix='arch -arm64' - # ARM Mac requires a firestore.json to specify the host as 127.0.0.1 rather than localhost. - # Otherwise the Firestore emulator cannot connect, probably because localhost is ipv6. - echo '{"emulators":{"firestore":{"port":"8080","host":"127.0.0.1"}}}' > firebase.json + # This is an M1 Mac GitHub self-hosted runner. + + # Use arch command to ensure that the app is running in the correct processor mode (arm64). + cmd_prefix='arch -${{ matrix.arch }}' + + # If Firestore is included, use the emulator; otherwise just run the tests standalone. + if [[ "${{ needs.check_and_prepare.outputs.apis }}" == *"firestore"* ]]; then + # ARM Mac requires a firestore.json to specify the host as 127.0.0.1 rather than localhost. + # Otherwise the Firestore emulator cannot connect, probably because localhost is ipv6. + echo '{"emulators":{"firestore":{"port":"8080","host":"127.0.0.1"}}}' > firebase.json + + set -x + firebase emulators:exec --only firestore --project demo-example \ + 'python scripts/gha/desktop_tester.py --testapp_dir testapps --logfile_name "desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}" --cmd_prefix='"\"${cmd_prefix}\"" + else + set -x + python scripts/gha/desktop_tester.py --testapp_dir testapps --logfile_name "desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}" --cmd_prefix="${cmd_prefix}" + fi fi - set -x - ${cmd_prefix} firebase emulators:exec --only firestore --project demo-example 'python scripts/gha/desktop_tester.py --testapp_dir testapps --logfile_name "desktop-${{ matrix.os }}-${{ matrix.arch}}-${{ matrix.ssl_variant }}"' - set +x env: USE_FIRESTORE_EMULATOR: true - name: Delete keychain (macOS) diff --git a/scripts/gha/desktop_tester.py b/scripts/gha/desktop_tester.py index eb7e4806b..5ec302bcd 100644 --- a/scripts/gha/desktop_tester.py +++ b/scripts/gha/desktop_tester.py @@ -44,7 +44,9 @@ "logfile_name", "", "Create test log artifact test-results-$logfile_name.log." " logfile will be created and placed in testapp_dir.") - +flags.DEFINE_string( + "cmd_prefix", "", + "Prefix to include before the command when running each test") def main(argv): if len(argv) > 1: @@ -106,7 +108,7 @@ def run(self): os.chmod(self.testapp_path, 0o777) try: result = subprocess.run( - args=[self.testapp_path], + args=FLAGS.cmd_prefix.split() + [self.testapp_path], cwd=os.path.dirname(self.testapp_path), # Testapp uses CWD for config stdout=subprocess.PIPE, stderr=subprocess.STDOUT, From 14ac12cf52c0d9ef7a36f8fc0a52157319650a2d Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sat, 7 May 2022 18:36:36 -0700 Subject: [PATCH 57/88] Change simulator device --- scripts/gha/print_matrix_configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gha/print_matrix_configuration.py b/scripts/gha/print_matrix_configuration.py index a80e68266..4a4dc0597 100644 --- a/scripts/gha/print_matrix_configuration.py +++ b/scripts/gha/print_matrix_configuration.py @@ -170,7 +170,7 @@ "ios_min": {"type": "real", "model":"iphone8", "version":"11.4"}, "ios_target": {"type": "real", "model":"iphone8plus", "version":"12.0"}, "ios_latest": {"type": "real", "model":"iphone11pro", "version":"14.7"}, - "simulator_min": {"type": "virtual", "name":"iPhone 6", "version":"13.7"}, + "simulator_min": {"type": "virtual", "name":"iPhone 8", "version":"13.7"}, "simulator_target": {"type": "virtual", "name":"iPhone 8", "version":"14.5"}, "simulator_latest": {"type": "virtual", "name":"iPhone 11", "version":"15.2"}, "tvos_simulator": {"type": "virtual", "name":"Apple TV", "version":"14.3"}, From 3db31664ac4671725bf2ce4d5ca9d29e4b8ea8d4 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sun, 8 May 2022 11:43:10 -0700 Subject: [PATCH 58/88] Set up a toolchain file for 32-bit linux builds on packaged SDK --- scripts/gha/build_testapps.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/scripts/gha/build_testapps.py b/scripts/gha/build_testapps.py index e847a865e..5fb78cea5 100644 --- a/scripts/gha/build_testapps.py +++ b/scripts/gha/build_testapps.py @@ -73,14 +73,15 @@ """ +import attr import datetime +import json import os import platform import shutil import subprocess import sys -import json -import attr +import tempfile from absl import app from absl import flags @@ -285,6 +286,22 @@ def main(argv): "-DFIREBASE_PYTHON_HOST_EXECUTABLE:FILEPATH=%s" % sys.executable, )) + if (_DESKTOP in platforms and FLAGS.packaged_sdk and + utils.is_linux_os() and FLAGS.arch == "x86"): + # Write out a temporary toolchain file to force 32-bit Linux builds, as + # the SDK-included toolchain file may not be present when building against + # the packaged SDK. + temp_toolchain_file = tempfile.NamedTemporaryFile(suffix=".cmake") + temp_toolchain_file.writelines([ + 'set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32")', + 'set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32")', + 'set(CMAKE_LIBRARY_PATH "/usr/lib/i386-linux-gnu")', + 'set(INCLUDE_DIRECTORIES ${INCLUDE_DIRECTORIES} "/usr/include/i386-linux-gnu")']) + # Leave the file open, as it will be deleted on close, i.e. when this script exits. + # (On Linux, the file can be opened a second time by cmake while still open by + # this script) + cmake_flags.extend(["-DCMAKE_TOOLCHAIN_FILE=%s" % temp_toolchain_file.name]) + if FLAGS.cmake_flag: cmake_flags.extend(FLAGS.cmake_flag) @@ -480,6 +497,7 @@ def _build_desktop(sdk_dir, cmake_flags): # Ensure that correct Mac architecture is built. cmake_configure_cmd += ["-DCMAKE_OSX_ARCHITECTURES=%s" % ("arm64" if FLAGS.arch == "arm64" else "x86_64")] + _run(cmake_configure_cmd + cmake_flags) _run(["cmake", "--build", ".", "--config", "Debug"] + ["-j", str(FLAGS.jobs)] if FLAGS.jobs > 0 else []) From 09cf821cb56748cac4f2a850daa16ac6d6f918a2 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sun, 8 May 2022 13:32:08 -0700 Subject: [PATCH 59/88] Write file non-binary --- scripts/gha/build_testapps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gha/build_testapps.py b/scripts/gha/build_testapps.py index 5fb78cea5..6605bf637 100644 --- a/scripts/gha/build_testapps.py +++ b/scripts/gha/build_testapps.py @@ -291,7 +291,7 @@ def main(argv): # Write out a temporary toolchain file to force 32-bit Linux builds, as # the SDK-included toolchain file may not be present when building against # the packaged SDK. - temp_toolchain_file = tempfile.NamedTemporaryFile(suffix=".cmake") + temp_toolchain_file = tempfile.NamedTemporaryFile("w", suffix=".cmake") temp_toolchain_file.writelines([ 'set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32")', 'set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32")', From 3b1093539b3c0b7d0b1c73e6648369fb64593a34 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sun, 8 May 2022 13:58:35 -0700 Subject: [PATCH 60/88] Try another way of specifying runner --- .github/workflows/integration_tests.yml | 3 ++- scripts/gha/build_testapps.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 4be189285..1ed03840d 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -945,7 +945,7 @@ jobs: test_desktop_custom_runners: name: test-desktop-custom-runner-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} needs: [check_and_prepare, build_desktop] - runs-on: [self-hosted, firebase-cpp, '${{ matrix.runner_label }}'] + runs-on: fromJSON(matrix.runs_on_json) if: contains(needs.check_and_prepare.outputs.matrix_platform, 'Desktop') && contains(needs.check_and_prepare.outputs.matrix_arch_combined, 'arm64') && needs.check_and_prepare.outputs.apis != '' && !cancelled() strategy: fail-fast: false @@ -954,6 +954,7 @@ jobs: os: ${{ fromJson(needs.check_and_prepare.outputs.matrix_os) }} arch: ${{ fromJson(needs.check_and_prepare.outputs.matrix_arch_combined) }} runner_label: ['${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }}'] + runs_on_json: ['["self-hosted", "firebase-cpp", "${{ matrix.runner_label }}"]'] exclude: # Don't run Linux or Windows tests on the Mac arm64 runner. - os: ubuntu-latest diff --git a/scripts/gha/build_testapps.py b/scripts/gha/build_testapps.py index 6605bf637..a0f47dd2c 100644 --- a/scripts/gha/build_testapps.py +++ b/scripts/gha/build_testapps.py @@ -291,7 +291,7 @@ def main(argv): # Write out a temporary toolchain file to force 32-bit Linux builds, as # the SDK-included toolchain file may not be present when building against # the packaged SDK. - temp_toolchain_file = tempfile.NamedTemporaryFile("w", suffix=".cmake") + temp_toolchain_file = tempfile.NamedTemporaryFile("w+", suffix=".cmake") temp_toolchain_file.writelines([ 'set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32")', 'set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32")', From d6ea9c8380227098789de0c700f214dd353e0013 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sun, 8 May 2022 14:00:51 -0700 Subject: [PATCH 61/88] Try another way --- .github/workflows/integration_tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 1ed03840d..be9bc747b 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -945,7 +945,7 @@ jobs: test_desktop_custom_runners: name: test-desktop-custom-runner-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} needs: [check_and_prepare, build_desktop] - runs-on: fromJSON(matrix.runs_on_json) + runs-on: [self-hosted, firebase-cpp] + [${{ matrix.runner_label }}] if: contains(needs.check_and_prepare.outputs.matrix_platform, 'Desktop') && contains(needs.check_and_prepare.outputs.matrix_arch_combined, 'arm64') && needs.check_and_prepare.outputs.apis != '' && !cancelled() strategy: fail-fast: false @@ -954,7 +954,6 @@ jobs: os: ${{ fromJson(needs.check_and_prepare.outputs.matrix_os) }} arch: ${{ fromJson(needs.check_and_prepare.outputs.matrix_arch_combined) }} runner_label: ['${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }}'] - runs_on_json: ['["self-hosted", "firebase-cpp", "${{ matrix.runner_label }}"]'] exclude: # Don't run Linux or Windows tests on the Mac arm64 runner. - os: ubuntu-latest From 87dd93de1889cad11f1b145d0c5b7e0c3e6503de Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sun, 8 May 2022 14:01:36 -0700 Subject: [PATCH 62/88] Put back --- .github/workflows/integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index be9bc747b..4be189285 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -945,7 +945,7 @@ jobs: test_desktop_custom_runners: name: test-desktop-custom-runner-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} needs: [check_and_prepare, build_desktop] - runs-on: [self-hosted, firebase-cpp] + [${{ matrix.runner_label }}] + runs-on: [self-hosted, firebase-cpp, '${{ matrix.runner_label }}'] if: contains(needs.check_and_prepare.outputs.matrix_platform, 'Desktop') && contains(needs.check_and_prepare.outputs.matrix_arch_combined, 'arm64') && needs.check_and_prepare.outputs.apis != '' && !cancelled() strategy: fail-fast: false From 938b07174c472ac0eaa4178f04884915a82b3662 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sun, 8 May 2022 14:03:21 -0700 Subject: [PATCH 63/88] Only run custom runner if macos --- .github/workflows/integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 4be189285..98669c5bd 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -946,7 +946,7 @@ jobs: name: test-desktop-custom-runner-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} needs: [check_and_prepare, build_desktop] runs-on: [self-hosted, firebase-cpp, '${{ matrix.runner_label }}'] - if: contains(needs.check_and_prepare.outputs.matrix_platform, 'Desktop') && contains(needs.check_and_prepare.outputs.matrix_arch_combined, 'arm64') && needs.check_and_prepare.outputs.apis != '' && !cancelled() + if: contains(needs.check_and_prepare.outputs.matrix_platform, 'Desktop') && contains(needs.check_and_prepare.outputs.matrix_arch_combined, 'arm64') && contains(needs.check_and_prepare.outputs.matrix_os, 'macos-12') && needs.check_and_prepare.outputs.apis != '' && !cancelled() strategy: fail-fast: false matrix: From 096c8c77dffd8bc5455437250f1c59224b1aef12 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sun, 8 May 2022 14:30:30 -0700 Subject: [PATCH 64/88] Flush file. --- scripts/gha/build_testapps.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/gha/build_testapps.py b/scripts/gha/build_testapps.py index a0f47dd2c..7bf04616d 100644 --- a/scripts/gha/build_testapps.py +++ b/scripts/gha/build_testapps.py @@ -293,10 +293,11 @@ def main(argv): # the packaged SDK. temp_toolchain_file = tempfile.NamedTemporaryFile("w+", suffix=".cmake") temp_toolchain_file.writelines([ - 'set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32")', - 'set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32")', - 'set(CMAKE_LIBRARY_PATH "/usr/lib/i386-linux-gnu")', - 'set(INCLUDE_DIRECTORIES ${INCLUDE_DIRECTORIES} "/usr/include/i386-linux-gnu")']) + 'set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32")\n', + 'set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32")\n', + 'set(CMAKE_LIBRARY_PATH "/usr/lib/i386-linux-gnu")\n', + 'set(INCLUDE_DIRECTORIES ${INCLUDE_DIRECTORIES} "/usr/include/i386-linux-gnu")\n']) + temp_toolchain_file.writelines() # Leave the file open, as it will be deleted on close, i.e. when this script exits. # (On Linux, the file can be opened a second time by cmake while still open by # this script) From 4fe6f71ef750aece3f546aa0d119a8654b4ca182 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sun, 8 May 2022 14:31:34 -0700 Subject: [PATCH 65/88] Flush for real --- scripts/gha/build_testapps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gha/build_testapps.py b/scripts/gha/build_testapps.py index 7bf04616d..1fea62107 100644 --- a/scripts/gha/build_testapps.py +++ b/scripts/gha/build_testapps.py @@ -297,7 +297,7 @@ def main(argv): 'set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32")\n', 'set(CMAKE_LIBRARY_PATH "/usr/lib/i386-linux-gnu")\n', 'set(INCLUDE_DIRECTORIES ${INCLUDE_DIRECTORIES} "/usr/include/i386-linux-gnu")\n']) - temp_toolchain_file.writelines() + temp_toolchain_file.flush() # Leave the file open, as it will be deleted on close, i.e. when this script exits. # (On Linux, the file can be opened a second time by cmake while still open by # this script) From bf6d30c824cdc76410f4d5891badc777176f5a6e Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sun, 8 May 2022 14:42:12 -0700 Subject: [PATCH 66/88] Fix x86 Linux build directory for Linux hosts --- release_build_files/CMakeLists.txt | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/release_build_files/CMakeLists.txt b/release_build_files/CMakeLists.txt index fae75b12e..5a2088487 100644 --- a/release_build_files/CMakeLists.txt +++ b/release_build_files/CMakeLists.txt @@ -46,7 +46,7 @@ elseif(MSVC) set(MSVC_VS_VERSION VS2019) set(FIREBASE_SDK_LIBDIR ${FIREBASE_CPP_SDK_DIR}/libs/windows/${MSVC_VS_VERSION}/${MSVC_RUNTIME_MODE}/${MSVC_CPU}/${MSVC_CONFIG}) -else() +else() # LINUX # Check whether we are building with CXX11 ABI. get_directory_property(CURR_DIRECTORY_DEFS COMPILE_DEFINITIONS) if ("${CURR_DIRECTORY_DEFS}" MATCHES "_GLIBCXX_USE_CXX11_ABI=0" OR @@ -56,8 +56,13 @@ else() else() set(DISABLE_CXX11 FALSE) set(LINUX_ABI cxx11) - endif() - set(LINUX_CPU x86_64) + endif() + # Check if we are building for x86 + if ("${CMAKE_CXX_FLAGS}" MATCHES "-m32") + set(LINUX_CPU i386) + else() + set(LINUX_CPU x86_64) + endif() set(FIREBASE_SDK_LIBDIR ${FIREBASE_CPP_SDK_DIR}/libs/linux/${LINUX_CPU}/${LINUX_ABI}) endif() From ddcc3262fc7881dde1b574a06e9d59a065986e39 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sun, 8 May 2022 18:22:54 -0700 Subject: [PATCH 67/88] Replace jcenter(), long deprecated, with mavenCentral(). --- admob/admob_resources/build.gradle | 4 ++-- admob/build.gradle | 4 ++-- admob/integration_test/build.gradle | 4 ++-- analytics/build.gradle | 4 ++-- analytics/integration_test/build.gradle | 4 ++-- app/app_resources/build.gradle | 4 ++-- app/build.gradle | 4 ++-- app/google_api_resources/build.gradle | 4 ++-- app/integration_test/build.gradle | 4 ++-- app/invites_resources/build.gradle | 4 ++-- app/test_resources/build.gradle | 4 ++-- auth/auth_resources/build.gradle | 4 ++-- auth/build.gradle | 4 ++-- auth/integration_test/build.gradle | 4 ++-- database/build.gradle | 4 ++-- database/database_resources/build.gradle | 4 ++-- database/integration_test/build.gradle | 4 ++-- dynamic_links/build.gradle | 4 ++-- dynamic_links/integration_test/build.gradle | 4 ++-- firestore/build.gradle | 4 ++-- firestore/firestore_resources/build.gradle | 4 ++-- firestore/integration_test/build.gradle | 4 ++-- firestore/integration_test_internal/build.gradle | 4 ++-- functions/build.gradle | 4 ++-- functions/integration_test/build.gradle | 4 ++-- installations/build.gradle | 4 ++-- installations/integration_test/build.gradle | 4 ++-- messaging/build.gradle | 4 ++-- messaging/integration_test/build.gradle | 4 ++-- messaging/messaging_java/build.gradle | 4 ++-- remote_config/build.gradle | 4 ++-- remote_config/integration_test/build.gradle | 4 ++-- scripts/gha/integration_testing/gameloop_android/build.gradle | 4 ++-- storage/build.gradle | 4 ++-- storage/integration_test/build.gradle | 4 ++-- storage/storage_resources/build.gradle | 4 ++-- testing/build.gradle | 4 ++-- 37 files changed, 74 insertions(+), 74 deletions(-) diff --git a/admob/admob_resources/build.gradle b/admob/admob_resources/build.gradle index d80bc8f4a..533dc5d3e 100644 --- a/admob/admob_resources/build.gradle +++ b/admob/admob_resources/build.gradle @@ -15,7 +15,7 @@ buildscript { repositories { google() - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -25,7 +25,7 @@ buildscript { allprojects { repositories { google() - jcenter() + mavenCentral() } } diff --git a/admob/build.gradle b/admob/build.gradle index f1b5a00c5..b3aa6e271 100644 --- a/admob/build.gradle +++ b/admob/build.gradle @@ -15,7 +15,7 @@ buildscript { repositories { google() - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -24,7 +24,7 @@ buildscript { allprojects { repositories { google() - jcenter() + mavenCentral() } } diff --git a/admob/integration_test/build.gradle b/admob/integration_test/build.gradle index de7a6f1e4..11d440b8e 100644 --- a/admob/integration_test/build.gradle +++ b/admob/integration_test/build.gradle @@ -17,7 +17,7 @@ buildscript { repositories { mavenLocal() maven { url 'https://maven.google.com' } - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -29,7 +29,7 @@ allprojects { repositories { mavenLocal() maven { url 'https://maven.google.com' } - jcenter() + mavenCentral() } } diff --git a/analytics/build.gradle b/analytics/build.gradle index b10304768..3409aeb9b 100644 --- a/analytics/build.gradle +++ b/analytics/build.gradle @@ -15,7 +15,7 @@ buildscript { repositories { google() - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -24,7 +24,7 @@ buildscript { allprojects { repositories { google() - jcenter() + mavenCentral() } } diff --git a/analytics/integration_test/build.gradle b/analytics/integration_test/build.gradle index e4dd2178e..735936445 100644 --- a/analytics/integration_test/build.gradle +++ b/analytics/integration_test/build.gradle @@ -17,7 +17,7 @@ buildscript { repositories { mavenLocal() maven { url 'https://maven.google.com' } - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -29,7 +29,7 @@ allprojects { repositories { mavenLocal() maven { url 'https://maven.google.com' } - jcenter() + mavenCentral() } } diff --git a/app/app_resources/build.gradle b/app/app_resources/build.gradle index 73b2f40cc..3a883d8eb 100644 --- a/app/app_resources/build.gradle +++ b/app/app_resources/build.gradle @@ -15,7 +15,7 @@ buildscript { repositories { google() - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -25,7 +25,7 @@ buildscript { allprojects { repositories { google() - jcenter() + mavenCentral() } } diff --git a/app/build.gradle b/app/build.gradle index 89edc14b3..2198421ef 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -15,7 +15,7 @@ buildscript { repositories { google() - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -24,7 +24,7 @@ buildscript { allprojects { repositories { google() - jcenter() + mavenCentral() } } diff --git a/app/google_api_resources/build.gradle b/app/google_api_resources/build.gradle index 910f16ff2..8575019e3 100644 --- a/app/google_api_resources/build.gradle +++ b/app/google_api_resources/build.gradle @@ -15,7 +15,7 @@ buildscript { repositories { google() - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -25,7 +25,7 @@ buildscript { allprojects { repositories { google() - jcenter() + mavenCentral() } } diff --git a/app/integration_test/build.gradle b/app/integration_test/build.gradle index 622410f58..20c9e2df2 100644 --- a/app/integration_test/build.gradle +++ b/app/integration_test/build.gradle @@ -17,7 +17,7 @@ buildscript { repositories { mavenLocal() maven { url 'https://maven.google.com' } - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -29,7 +29,7 @@ allprojects { repositories { mavenLocal() maven { url 'https://maven.google.com' } - jcenter() + mavenCentral() } } diff --git a/app/invites_resources/build.gradle b/app/invites_resources/build.gradle index 23920e143..06669d0f1 100644 --- a/app/invites_resources/build.gradle +++ b/app/invites_resources/build.gradle @@ -15,7 +15,7 @@ buildscript { repositories { google() - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -25,7 +25,7 @@ buildscript { allprojects { repositories { google() - jcenter() + mavenCentral() } } diff --git a/app/test_resources/build.gradle b/app/test_resources/build.gradle index 16f2c7abe..2e145d15b 100644 --- a/app/test_resources/build.gradle +++ b/app/test_resources/build.gradle @@ -15,7 +15,7 @@ buildscript { repositories { google() - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -25,7 +25,7 @@ buildscript { allprojects { repositories { google() - jcenter() + mavenCentral() } } diff --git a/auth/auth_resources/build.gradle b/auth/auth_resources/build.gradle index ffb6dc106..5ce899afb 100644 --- a/auth/auth_resources/build.gradle +++ b/auth/auth_resources/build.gradle @@ -15,7 +15,7 @@ buildscript { repositories { google() - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -25,7 +25,7 @@ buildscript { allprojects { repositories { google() - jcenter() + mavenCentral() } } diff --git a/auth/build.gradle b/auth/build.gradle index 1874f9738..5938e531e 100644 --- a/auth/build.gradle +++ b/auth/build.gradle @@ -15,7 +15,7 @@ buildscript { repositories { google() - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -24,7 +24,7 @@ buildscript { allprojects { repositories { google() - jcenter() + mavenCentral() } } diff --git a/auth/integration_test/build.gradle b/auth/integration_test/build.gradle index 81497dfa2..461ae0982 100644 --- a/auth/integration_test/build.gradle +++ b/auth/integration_test/build.gradle @@ -17,7 +17,7 @@ buildscript { repositories { mavenLocal() maven { url 'https://maven.google.com' } - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -29,7 +29,7 @@ allprojects { repositories { mavenLocal() maven { url 'https://maven.google.com' } - jcenter() + mavenCentral() } } diff --git a/database/build.gradle b/database/build.gradle index bafe7785d..0399e831b 100644 --- a/database/build.gradle +++ b/database/build.gradle @@ -15,7 +15,7 @@ buildscript { repositories { google() - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -24,7 +24,7 @@ buildscript { allprojects { repositories { google() - jcenter() + mavenCentral() } } diff --git a/database/database_resources/build.gradle b/database/database_resources/build.gradle index d0d298ae0..d864bbe1a 100644 --- a/database/database_resources/build.gradle +++ b/database/database_resources/build.gradle @@ -15,7 +15,7 @@ buildscript { repositories { google() - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -25,7 +25,7 @@ buildscript { allprojects { repositories { google() - jcenter() + mavenCentral() } } diff --git a/database/integration_test/build.gradle b/database/integration_test/build.gradle index d78975819..31d51bd54 100644 --- a/database/integration_test/build.gradle +++ b/database/integration_test/build.gradle @@ -17,7 +17,7 @@ buildscript { repositories { mavenLocal() maven { url 'https://maven.google.com' } - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -29,7 +29,7 @@ allprojects { repositories { mavenLocal() maven { url 'https://maven.google.com' } - jcenter() + mavenCentral() } } diff --git a/dynamic_links/build.gradle b/dynamic_links/build.gradle index 570514f91..a19d2ba9a 100644 --- a/dynamic_links/build.gradle +++ b/dynamic_links/build.gradle @@ -15,7 +15,7 @@ buildscript { repositories { google() - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -24,7 +24,7 @@ buildscript { allprojects { repositories { google() - jcenter() + mavenCentral() } } diff --git a/dynamic_links/integration_test/build.gradle b/dynamic_links/integration_test/build.gradle index a5cf877d3..b4473efd4 100644 --- a/dynamic_links/integration_test/build.gradle +++ b/dynamic_links/integration_test/build.gradle @@ -17,7 +17,7 @@ buildscript { repositories { mavenLocal() maven { url 'https://maven.google.com' } - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -29,7 +29,7 @@ allprojects { repositories { mavenLocal() maven { url 'https://maven.google.com' } - jcenter() + mavenCentral() } } diff --git a/firestore/build.gradle b/firestore/build.gradle index a06792599..e9f0723f5 100644 --- a/firestore/build.gradle +++ b/firestore/build.gradle @@ -15,7 +15,7 @@ buildscript { repositories { google() - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -24,7 +24,7 @@ buildscript { allprojects { repositories { google() - jcenter() + mavenCentral() } } diff --git a/firestore/firestore_resources/build.gradle b/firestore/firestore_resources/build.gradle index afa0e0671..4a9f0085b 100644 --- a/firestore/firestore_resources/build.gradle +++ b/firestore/firestore_resources/build.gradle @@ -15,7 +15,7 @@ buildscript { repositories { google() - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -25,7 +25,7 @@ buildscript { allprojects { repositories { google() - jcenter() + mavenCentral() } } diff --git a/firestore/integration_test/build.gradle b/firestore/integration_test/build.gradle index 89215fce0..b44d7e03f 100644 --- a/firestore/integration_test/build.gradle +++ b/firestore/integration_test/build.gradle @@ -17,7 +17,7 @@ buildscript { repositories { mavenLocal() maven { url 'https://maven.google.com' } - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -29,7 +29,7 @@ allprojects { repositories { mavenLocal() maven { url 'https://maven.google.com' } - jcenter() + mavenCentral() } } diff --git a/firestore/integration_test_internal/build.gradle b/firestore/integration_test_internal/build.gradle index a8bbcbff1..25113987c 100644 --- a/firestore/integration_test_internal/build.gradle +++ b/firestore/integration_test_internal/build.gradle @@ -17,7 +17,7 @@ buildscript { repositories { mavenLocal() maven { url 'https://maven.google.com' } - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -29,7 +29,7 @@ allprojects { repositories { mavenLocal() maven { url 'https://maven.google.com' } - jcenter() + mavenCentral() } } diff --git a/functions/build.gradle b/functions/build.gradle index 0376db083..55da3b4d2 100644 --- a/functions/build.gradle +++ b/functions/build.gradle @@ -15,7 +15,7 @@ buildscript { repositories { google() - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -24,7 +24,7 @@ buildscript { allprojects { repositories { google() - jcenter() + mavenCentral() } } diff --git a/functions/integration_test/build.gradle b/functions/integration_test/build.gradle index 7617f84b0..cd2f8be3d 100644 --- a/functions/integration_test/build.gradle +++ b/functions/integration_test/build.gradle @@ -17,7 +17,7 @@ buildscript { repositories { mavenLocal() maven { url 'https://maven.google.com' } - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -29,7 +29,7 @@ allprojects { repositories { mavenLocal() maven { url 'https://maven.google.com' } - jcenter() + mavenCentral() } } diff --git a/installations/build.gradle b/installations/build.gradle index 123c87465..58fd6f1f4 100644 --- a/installations/build.gradle +++ b/installations/build.gradle @@ -15,7 +15,7 @@ buildscript { repositories { google() - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -24,7 +24,7 @@ buildscript { allprojects { repositories { google() - jcenter() + mavenCentral() } } diff --git a/installations/integration_test/build.gradle b/installations/integration_test/build.gradle index 82899da6e..835497cbf 100644 --- a/installations/integration_test/build.gradle +++ b/installations/integration_test/build.gradle @@ -17,7 +17,7 @@ buildscript { repositories { mavenLocal() maven { url 'https://maven.google.com' } - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -29,7 +29,7 @@ allprojects { repositories { mavenLocal() maven { url 'https://maven.google.com' } - jcenter() + mavenCentral() } } diff --git a/messaging/build.gradle b/messaging/build.gradle index 238265ca3..a9bf5ae9b 100644 --- a/messaging/build.gradle +++ b/messaging/build.gradle @@ -15,7 +15,7 @@ buildscript { repositories { google() - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -24,7 +24,7 @@ buildscript { allprojects { repositories { google() - jcenter() + mavenCentral() } } diff --git a/messaging/integration_test/build.gradle b/messaging/integration_test/build.gradle index ae7e118f2..86f5a2f31 100644 --- a/messaging/integration_test/build.gradle +++ b/messaging/integration_test/build.gradle @@ -17,7 +17,7 @@ buildscript { repositories { mavenLocal() maven { url 'https://maven.google.com' } - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -29,7 +29,7 @@ allprojects { repositories { mavenLocal() maven { url 'https://maven.google.com' } - jcenter() + mavenCentral() } } diff --git a/messaging/messaging_java/build.gradle b/messaging/messaging_java/build.gradle index 4106ef924..975cce2a9 100644 --- a/messaging/messaging_java/build.gradle +++ b/messaging/messaging_java/build.gradle @@ -15,7 +15,7 @@ buildscript { repositories { google() - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -30,7 +30,7 @@ buildscript { allprojects { repositories { google() - jcenter() + mavenCentral() } } diff --git a/remote_config/build.gradle b/remote_config/build.gradle index 35c179001..06f9dd84b 100644 --- a/remote_config/build.gradle +++ b/remote_config/build.gradle @@ -15,7 +15,7 @@ buildscript { repositories { google() - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -24,7 +24,7 @@ buildscript { allprojects { repositories { google() - jcenter() + mavenCentral() } } diff --git a/remote_config/integration_test/build.gradle b/remote_config/integration_test/build.gradle index 9307ce4a8..3166aa9b9 100644 --- a/remote_config/integration_test/build.gradle +++ b/remote_config/integration_test/build.gradle @@ -17,7 +17,7 @@ buildscript { repositories { mavenLocal() maven { url 'https://maven.google.com' } - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -29,7 +29,7 @@ allprojects { repositories { mavenLocal() maven { url 'https://maven.google.com' } - jcenter() + mavenCentral() } } diff --git a/scripts/gha/integration_testing/gameloop_android/build.gradle b/scripts/gha/integration_testing/gameloop_android/build.gradle index fbb346504..9c0ad89bc 100644 --- a/scripts/gha/integration_testing/gameloop_android/build.gradle +++ b/scripts/gha/integration_testing/gameloop_android/build.gradle @@ -3,7 +3,7 @@ buildscript { ext.kotlin_version = "1.3.72" repositories { google() - jcenter() + mavenCentral() } dependencies { classpath "com.android.tools.build:gradle:4.0.1" @@ -17,7 +17,7 @@ buildscript { allprojects { repositories { google() - jcenter() + mavenCentral() } } diff --git a/storage/build.gradle b/storage/build.gradle index 0e3b9a0da..095ddfd52 100644 --- a/storage/build.gradle +++ b/storage/build.gradle @@ -15,7 +15,7 @@ buildscript { repositories { google() - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -24,7 +24,7 @@ buildscript { allprojects { repositories { google() - jcenter() + mavenCentral() } } diff --git a/storage/integration_test/build.gradle b/storage/integration_test/build.gradle index 8616c8d87..40690730c 100644 --- a/storage/integration_test/build.gradle +++ b/storage/integration_test/build.gradle @@ -17,7 +17,7 @@ buildscript { repositories { mavenLocal() maven { url 'https://maven.google.com' } - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -29,7 +29,7 @@ allprojects { repositories { mavenLocal() maven { url 'https://maven.google.com' } - jcenter() + mavenCentral() } } diff --git a/storage/storage_resources/build.gradle b/storage/storage_resources/build.gradle index ceda73e2b..f40b419b5 100644 --- a/storage/storage_resources/build.gradle +++ b/storage/storage_resources/build.gradle @@ -15,7 +15,7 @@ buildscript { repositories { google() - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -25,7 +25,7 @@ buildscript { allprojects { repositories { google() - jcenter() + mavenCentral() } } diff --git a/testing/build.gradle b/testing/build.gradle index 7ce929f78..e753c21dd 100644 --- a/testing/build.gradle +++ b/testing/build.gradle @@ -15,7 +15,7 @@ buildscript { repositories { google() - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.3.3' @@ -27,7 +27,7 @@ buildscript { allprojects { repositories { google() - jcenter() + mavenCentral() } } From 660878abf7c97e0a2368aa6d3cbb64c1c93eaf00 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sun, 8 May 2022 20:43:06 -0700 Subject: [PATCH 68/88] Try to fix RTDB 32-bit bug --- database/integration_test/src/integration_test.cc | 4 ++-- database/src/desktop/core/server_values.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/database/integration_test/src/integration_test.cc b/database/integration_test/src/integration_test.cc index 139f63a48..beade7383 100644 --- a/database/integration_test/src/integration_test.cc +++ b/database/integration_test/src/integration_test.cc @@ -454,8 +454,8 @@ TEST_F(FirebaseDatabaseTest, TestSetAndGetSimpleValues) { WaitForCompletion(f7, "GetLongDouble"); // Get the current time to compare to the Timestamp. - int64_t current_time_milliseconds = - static_cast(time(nullptr)) * 1000L; + time_t current_time = time(nullptr) + int64_t current_time_milliseconds = current_time * 1000L; EXPECT_EQ(f1.result()->value().AsString(), kSimpleString); EXPECT_EQ(f2.result()->value().AsInt64(), kSimpleInt); diff --git a/database/src/desktop/core/server_values.cc b/database/src/desktop/core/server_values.cc index 235cb8fff..349d28a99 100644 --- a/database/src/desktop/core/server_values.cc +++ b/database/src/desktop/core/server_values.cc @@ -28,7 +28,7 @@ static const char kNameSubkeyServerValue[] = ".sv"; Variant GenerateServerValues(int64_t server_time_offset) { Variant server_values = Variant::EmptyMap(); - int64_t corrected_time = time(nullptr) * 1000L + server_time_offset; + int64_t corrected_time = static_cast(time(nullptr)) * 1000L + server_time_offset; server_values.map()["timestamp"] = Variant::FromInt64(corrected_time); return server_values; } From ee6573aa9e4468407696fb14c218d97310e02c58 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sun, 8 May 2022 22:18:59 -0700 Subject: [PATCH 69/88] Restore test back --- database/integration_test/src/integration_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database/integration_test/src/integration_test.cc b/database/integration_test/src/integration_test.cc index beade7383..139f63a48 100644 --- a/database/integration_test/src/integration_test.cc +++ b/database/integration_test/src/integration_test.cc @@ -454,8 +454,8 @@ TEST_F(FirebaseDatabaseTest, TestSetAndGetSimpleValues) { WaitForCompletion(f7, "GetLongDouble"); // Get the current time to compare to the Timestamp. - time_t current_time = time(nullptr) - int64_t current_time_milliseconds = current_time * 1000L; + int64_t current_time_milliseconds = + static_cast(time(nullptr)) * 1000L; EXPECT_EQ(f1.result()->value().AsString(), kSimpleString); EXPECT_EQ(f2.result()->value().AsInt64(), kSimpleInt); From 94ca12563eec4687ecc8b54320b6be4451a875d2 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sun, 8 May 2022 22:51:45 -0700 Subject: [PATCH 70/88] Clarify argument help text --- scripts/gha/install_prereqs_desktop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gha/install_prereqs_desktop.py b/scripts/gha/install_prereqs_desktop.py index 756ad7472..65171c022 100644 --- a/scripts/gha/install_prereqs_desktop.py +++ b/scripts/gha/install_prereqs_desktop.py @@ -96,7 +96,7 @@ def parse_cmdline_args(): parser = argparse.ArgumentParser(description='Install prerequisites for building cpp sdk') parser.add_argument('--arch', default=None, help='Install support libraries to build a specific architecture (currently supported: x86)') parser.add_argument('--running_only', action='store_true', help='Only install prerequisites for running, not for building') - parser.add_argument('--gha_build', action='store_true', default=None, help='Set to true when building on GitHub, for metric tracking purposes (also changes some prerequisite installation behavior).') + parser.add_argument('--gha_build', action='store_true', default=None, help='Set to true when building on GitHub, changing some prerequisite installation behavior.') args = parser.parse_args() return args From 9b14791a1345c57032fa0f112ec69dac471e78c9 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sun, 8 May 2022 22:54:22 -0700 Subject: [PATCH 71/88] Make cmake CPU check more accurate --- release_build_files/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release_build_files/CMakeLists.txt b/release_build_files/CMakeLists.txt index 5a2088487..431192627 100644 --- a/release_build_files/CMakeLists.txt +++ b/release_build_files/CMakeLists.txt @@ -58,7 +58,7 @@ else() # LINUX set(LINUX_ABI cxx11) endif() # Check if we are building for x86 - if ("${CMAKE_CXX_FLAGS}" MATCHES "-m32") + if (" ${CMAKE_CXX_FLAGS} " MATCHES " -m32 ") set(LINUX_CPU i386) else() set(LINUX_CPU x86_64) From 9c5e52f586f573910b6e1ce24095540247c6b5e0 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sun, 8 May 2022 23:06:01 -0700 Subject: [PATCH 72/88] Clean up job specification --- .github/workflows/integration_tests.yml | 34 ++++++++++++------------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 98669c5bd..c2e3038d5 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -1013,28 +1013,26 @@ jobs: python scripts/gha/restore_secrets.py --passphrase_file=- --artifact testapps < firebase.json + # If Firestore is included, use the emulator; otherwise just run the tests standalone. + if [[ "${{ needs.check_and_prepare.outputs.apis }}" == *"firestore"* ]]; then + # ARM Mac requires a firestore.json to specify the host as 127.0.0.1 rather than localhost. + # Otherwise the Firestore emulator cannot connect, probably because localhost is ipv6. + echo '{"emulators":{"firestore":{"port":"8080","host":"127.0.0.1"}}}' > firebase.json - set -x - firebase emulators:exec --only firestore --project demo-example \ - 'python scripts/gha/desktop_tester.py --testapp_dir testapps --logfile_name "desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}" --cmd_prefix='"\"${cmd_prefix}\"" - else - set -x - python scripts/gha/desktop_tester.py --testapp_dir testapps --logfile_name "desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}" --cmd_prefix="${cmd_prefix}" - fi + set -x + firebase emulators:exec --only firestore --project demo-example \ + 'python scripts/gha/desktop_tester.py --testapp_dir testapps --logfile_name "desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}" --cmd_prefix='"\"${cmd_prefix}\"" + else + set -x + python scripts/gha/desktop_tester.py --testapp_dir testapps --logfile_name "desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}" --cmd_prefix="${cmd_prefix}" fi env: USE_FIRESTORE_EMULATOR: true From 6f445cc92a411dc4fe49927493d0eafb5be8f636 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Mon, 9 May 2022 00:41:17 -0700 Subject: [PATCH 73/88] Format code. --- database/src/desktop/core/server_values.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/database/src/desktop/core/server_values.cc b/database/src/desktop/core/server_values.cc index 349d28a99..34a5c803e 100644 --- a/database/src/desktop/core/server_values.cc +++ b/database/src/desktop/core/server_values.cc @@ -28,7 +28,8 @@ static const char kNameSubkeyServerValue[] = ".sv"; Variant GenerateServerValues(int64_t server_time_offset) { Variant server_values = Variant::EmptyMap(); - int64_t corrected_time = static_cast(time(nullptr)) * 1000L + server_time_offset; + int64_t corrected_time = + static_cast(time(nullptr)) * 1000L + server_time_offset; server_values.map()["timestamp"] = Variant::FromInt64(corrected_time); return server_values; } From 80e40606b402bed70e323a4bc1db846ed3f37068 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Mon, 9 May 2022 01:32:22 -0700 Subject: [PATCH 74/88] Force bash shell when updating PR comment --- .github/workflows/integration_tests.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index c2e3038d5..cac03f140 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -218,6 +218,7 @@ jobs: echo "::set-output name=tvos_device::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k tvos_device -t ${mobile_test_on} )" - name: Update PR label and comment if: steps.set_outputs.outputs.pr_number + shell: bash run: | #Add the in-progress label and remove any previous labels. python scripts/gha/it_workflow.py --stage start \ @@ -428,8 +429,8 @@ jobs: path: test_results name: log-artifact - name: Update PR label and comment - shell: bash if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }} + shell: bash run: | pushd ${{env.GCS_UPLOAD_DIR}} python scripts/gha/it_workflow.py --stage progress \ @@ -560,6 +561,7 @@ jobs: name: log-artifact - name: Update PR label and comment if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }} + shell: bash run: | python scripts/gha/it_workflow.py --stage progress \ --token ${{github.token}} \ @@ -680,6 +682,7 @@ jobs: name: log-artifact - name: Update PR label and comment if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }} + shell: bash run: | python scripts/gha/it_workflow.py --stage progress \ --token ${{github.token}} \ @@ -799,6 +802,7 @@ jobs: name: log-artifact - name: Update PR label and comment if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }} + shell: bash run: | python scripts/gha/it_workflow.py --stage progress \ --token ${{github.token}} \ @@ -924,6 +928,7 @@ jobs: name: log-artifact - name: Update PR label and comment if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }} + shell: bash run: | python scripts/gha/it_workflow.py --stage progress \ --token ${{github.token}} \ @@ -1067,6 +1072,7 @@ jobs: name: log-artifact - name: Update PR label and comment if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }} + shell: bash run: | python scripts/gha/it_workflow.py --stage progress \ --token ${{github.token}} \ @@ -1180,6 +1186,7 @@ jobs: name: log-artifact - name: Update PR label and comment if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }} + shell: bash run: | python scripts/gha/it_workflow.py --stage progress \ --token ${{github.token}} \ @@ -1285,6 +1292,7 @@ jobs: name: log-artifact - name: Update PR label and comment if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }} + shell: bash run: | python scripts/gha/it_workflow.py --stage progress \ --token ${{github.token}} \ @@ -1368,6 +1376,7 @@ jobs: name: log-artifact - name: Update PR label and comment if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }} + shell: bash run: | python scripts/gha/it_workflow.py --stage progress \ --token ${{github.token}} \ @@ -1414,6 +1423,7 @@ jobs: private_key: ${{ secrets.WORKFLOW_TRIGGER_APP_PRIVATE_KEY }} - name: Update PR label and comment if: ${{ needs.check_and_prepare.outputs.pr_number }} + shell: bash run: | python scripts/gha/it_workflow.py --stage end \ --token ${{github.token}} \ From fe0cba7fa509f7f6f209b548d8b256c1dee9c33f Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Mon, 9 May 2022 10:22:49 -0700 Subject: [PATCH 75/88] Rename job to make them easier to parse visually --- .github/workflows/integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index cac03f140..a9484b429 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -948,7 +948,7 @@ jobs: # Run tests that depend on custom (self-hosted) runners. # For now, this is only used for ARM Mac builds. test_desktop_custom_runners: - name: test-desktop-custom-runner-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }} + name: test-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}-custom-runner needs: [check_and_prepare, build_desktop] runs-on: [self-hosted, firebase-cpp, '${{ matrix.runner_label }}'] if: contains(needs.check_and_prepare.outputs.matrix_platform, 'Desktop') && contains(needs.check_and_prepare.outputs.matrix_arch_combined, 'arm64') && contains(needs.check_and_prepare.outputs.matrix_os, 'macos-12') && needs.check_and_prepare.outputs.apis != '' && !cancelled() From 26ec7b5c9aa9e642898179b2374ca347e89cb252 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Mon, 9 May 2022 10:33:07 -0700 Subject: [PATCH 76/88] Rename variable to be consistent with GitHub os name. --- .github/workflows/integration_tests.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index a9484b429..fe366ffb3 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -72,7 +72,7 @@ jobs: ios_device: ${{ steps.matrix_config.outputs.ios_device }} tvos_device: ${{ steps.matrix_config.outputs.tvos_device }} # Copy the runner label here because matrix specifiers cannot see env. - runner_label_mac_arm64: ${{ env.runnerLabelMacArm64 }} + runner_label_macos_arm64: ${{ env.runnerLabelMacArm64 }} steps: ### Fail the workflow if the user does not have admin access to run the tests. - name: Check if user has permission to trigger tests @@ -958,20 +958,20 @@ jobs: ssl_variant: ${{ fromJson(needs.check_and_prepare.outputs.matrix_ssl) }} os: ${{ fromJson(needs.check_and_prepare.outputs.matrix_os) }} arch: ${{ fromJson(needs.check_and_prepare.outputs.matrix_arch_combined) }} - runner_label: ['${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }}'] + runner_label: ['${{ needs.check_and_prepare.outputs.runner_label_macos_arm64 }}'] exclude: # Don't run Linux or Windows tests on the Mac arm64 runner. - os: ubuntu-latest - runner_label: ${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }} + runner_label: ${{ needs.check_and_prepare.outputs.runner_label_macos_arm64 }} - os: windows-latest - runner_label: ${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }} + runner_label: ${{ needs.check_and_prepare.outputs.runner_label_macos_arm64 }} # Don't run x86 tests on any Mac runner. - os: macos-12 arch: x86 # Don't run x64 tests on the Mac arm64 runner. - os: macos-12 arch: x64 - runner_label: ${{ needs.check_and_prepare.outputs.runner_label_mac_arm64 }} + runner_label: ${{ needs.check_and_prepare.outputs.runner_label_macos_arm64 }} # Until we support building openssl from source, we can't link to system # openssl when cross-compiling, so exclude openssl from mac arm64 testing. - os: macos-12 @@ -1019,7 +1019,7 @@ jobs: ${{ secrets.TEST_SECRET }} EOF - name: Run Desktop integration tests on M1 Mac - if: ${{ matrix.os == 'macos-12' && matrix.arch == 'arm64' && matrix.runner_label == needs.check_and_prepare.outputs.runner_label_mac_arm64 }} + if: ${{ matrix.os == 'macos-12' && matrix.arch == 'arm64' && matrix.runner_label == needs.check_and_prepare.outputs.runner_label_macos_arm64 }} shell: bash run: | # This is an M1 Mac GitHub self-hosted runner. From 0ce8afe82274aeaac6a9e5c930977e13c7fb9aeb Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Mon, 9 May 2022 11:49:37 -0700 Subject: [PATCH 77/88] Update packaging x86 to use new prereqs script option --- .github/workflows/cpp-packaging.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cpp-packaging.yml b/.github/workflows/cpp-packaging.yml index 165145a57..de845ba02 100644 --- a/.github/workflows/cpp-packaging.yml +++ b/.github/workflows/cpp-packaging.yml @@ -408,9 +408,13 @@ jobs: # binutils, used by older version of homebrew for hosting packages. brew update - - name: Install prerequisites - run: | - python scripts/gha/install_prereqs_desktop.py + - name: Install Desktop SDK prerequisites + uses: nick-invision/retry@v2 + with: + timeout_minutes: 15 + max_attempts: 3 + command: | + python scripts/gha/install_prereqs_desktop.py --gha_build --arch '${{ matrix.architecture }}' - name: Export verbose flag shell: bash From f9c6749800832b9f4032c350fbf0a08590cbdfa9 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Mon, 9 May 2022 11:57:11 -0700 Subject: [PATCH 78/88] Fix x86 CI builds to retry prereqs as well --- .github/workflows/desktop.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index 9891e3196..11176ae44 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -176,9 +176,13 @@ jobs: # binutils, used by older version of homebrew for hosting packages. brew update - - name: Install prerequisites - run: | - python scripts/gha/install_prereqs_desktop.py + - name: Install Desktop SDK prerequisites + uses: nick-invision/retry@v2 + with: + timeout_minutes: 15 + max_attempts: 3 + command: | + python scripts/gha/install_prereqs_desktop.py --gha_build --arch '${{ matrix.architecture }}' - name: Build SDK shell: bash From cb3cf90fb3319060064f4bff30b66729f6a7e4dd Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sat, 7 May 2022 10:56:49 -0700 Subject: [PATCH 79/88] Move install_x86_support_libraries to utils. --- scripts/gha/build_desktop.py | 2 +- scripts/gha/utils.py | 42 ++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/scripts/gha/build_desktop.py b/scripts/gha/build_desktop.py index 1978ff0ba..0a0090f81 100644 --- a/scripts/gha/build_desktop.py +++ b/scripts/gha/build_desktop.py @@ -281,7 +281,7 @@ def main(): # To build x86 on x86_64 linux hosts, we also need x86 support libraries if args.arch == 'x86' and utils.is_linux_os(): - install_x86_support_libraries(args.gha_build) + utils.install_x86_support_libraries(args.gha_build) # Install C++ dependencies using vcpkg if not args.disable_vcpkg: diff --git a/scripts/gha/utils.py b/scripts/gha/utils.py index 9201fed01..7f964e414 100644 --- a/scripts/gha/utils.py +++ b/scripts/gha/utils.py @@ -218,3 +218,45 @@ def clean_vcpkg_temp_data(): for directory_to_remove in directories_to_remove: abspath = os.path.join(vcpkg_root_dir_path, directory_to_remove) delete_directory(abspath) + + +def install_x86_support_libraries(gha_build=False): + """Install support libraries needed to build x86 on x86_64 hosts. + + Args: + gha_build: Pass in True if running on a GitHub runner; this will activate + workarounds that might be undesirable on a personal system (e.g. + downgrading Ubuntu packages). + """ + if is_linux_os(): + packages = ['gcc-multilib', 'g++-multilib', 'libglib2.0-dev:i386', + 'libsecret-1-dev:i386', 'libpthread-stubs0-dev:i386', + 'libssl-dev:i386'] + if gha_build: + # Workaround for GitHub runners, which have an incompatibility between the + # 64-bit and 32-bit versions of the Ubuntu package libpcre2-8-0. Downgrade + # the installed 64-bit version of the library to get around this issue. + # This will presumably be fixed in a future Ubuntu update. (If you remove + # it, remove the workaround further down this function as well.) + packages = ['--allow-downgrades'] + packages + ['libpcre2-8-0=10.34-7'] + + # First check if these packages exist on the machine already + devnull = open(os.devnull, "w") + process = subprocess.run(["dpkg", "-s"] + packages, stdout=devnull, stderr=subprocess.STDOUT) + devnull.close() + if process.returncode != 0: + # This implies not all of the required packages are already installed on user's machine + # Install them. + run_command(['dpkg', '--add-architecture', 'i386'], as_root=True, check=True) + run_command(['apt', 'update'], as_root=True, check=True) + run_command(['apt', 'install', '-V', '-y'] + packages, as_root=True, check=True) + + if gha_build: + # One more workaround: downgrading libpcre2-8-0 above may have uninstalled + # libsecret, which is required for the Linux build. Force it to be + # reinstalled, but do it as a separate command to ensure that held + # packages aren't modified. (Once the workaround above is removed, this can + # be removed as well.) + # Note: "-f" = "fix" - let apt do what it needs to do to fix dependencies. + run_command(['apt', 'install', '-f', '-V', '-y', 'libsecret-1-dev'], + as_root=True, check=True) From a6da2e647cf947f4a0811af76babcd91945504e8 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Mon, 9 May 2022 11:49:37 -0700 Subject: [PATCH 80/88] Update packaging x86 to use new prereqs script option --- .github/workflows/cpp-packaging.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cpp-packaging.yml b/.github/workflows/cpp-packaging.yml index 165145a57..de845ba02 100644 --- a/.github/workflows/cpp-packaging.yml +++ b/.github/workflows/cpp-packaging.yml @@ -408,9 +408,13 @@ jobs: # binutils, used by older version of homebrew for hosting packages. brew update - - name: Install prerequisites - run: | - python scripts/gha/install_prereqs_desktop.py + - name: Install Desktop SDK prerequisites + uses: nick-invision/retry@v2 + with: + timeout_minutes: 15 + max_attempts: 3 + command: | + python scripts/gha/install_prereqs_desktop.py --gha_build --arch '${{ matrix.architecture }}' - name: Export verbose flag shell: bash From bdf2d61ed8dee4ae1d3010aeb95816c622bac6a5 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Mon, 9 May 2022 11:57:11 -0700 Subject: [PATCH 81/88] Fix x86 CI builds to retry prereqs as well --- .github/workflows/desktop.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index 9891e3196..11176ae44 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -176,9 +176,13 @@ jobs: # binutils, used by older version of homebrew for hosting packages. brew update - - name: Install prerequisites - run: | - python scripts/gha/install_prereqs_desktop.py + - name: Install Desktop SDK prerequisites + uses: nick-invision/retry@v2 + with: + timeout_minutes: 15 + max_attempts: 3 + command: | + python scripts/gha/install_prereqs_desktop.py --gha_build --arch '${{ matrix.architecture }}' - name: Build SDK shell: bash From 381088ff415901addb8f82c440c60de40750b67e Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Mon, 9 May 2022 12:05:04 -0700 Subject: [PATCH 82/88] Add call to install_x86_support_libraries to prereqs script if needed. --- scripts/gha/install_prereqs_desktop.py | 118 ++++++++++++++----------- 1 file changed, 66 insertions(+), 52 deletions(-) diff --git a/scripts/gha/install_prereqs_desktop.py b/scripts/gha/install_prereqs_desktop.py index a8511c493..0cf4f54a5 100644 --- a/scripts/gha/install_prereqs_desktop.py +++ b/scripts/gha/install_prereqs_desktop.py @@ -29,62 +29,76 @@ """ +import argparse import utils def main(): - # Install protobuf on linux/mac if its not installed already - if not utils.is_command_installed('protoc'): - if utils.is_linux_os(): - # sudo apt install protobuf-compiler - utils.run_command(['apt', 'install', '-y','protobuf-compiler'], as_root=True) - elif utils.is_mac_os(): - # brew install protobuf - utils.run_command(['brew', 'install', 'protobuf']) - - # Install go on linux/mac if its not installed already - if not utils.is_command_installed('go'): - if utils.is_linux_os(): - # sudo apt install -y golang - utils.run_command(['apt', 'install', '-y','golang'], as_root=True) - elif utils.is_mac_os(): - # brew install protobuf - utils.run_command(['brew', 'install', 'go']) - - # Install openssl on linux/mac if its not installed already - if not utils.is_command_installed('go'): - if utils.is_linux_os(): - # sudo apt install -y openssl - utils.run_command(['apt', 'install', '-y','openssl'], as_root=True) - elif utils.is_mac_os(): - # brew install protobuf - utils.run_command(['brew', 'install', 'openssl']) - - # Install ccache on linux/mac if its not installed already - if not utils.is_command_installed('ccache'): - if utils.is_linux_os(): - # sudo apt install ccache - utils.run_command(['apt', 'install', '-y', 'ccache'], as_root=True) - elif utils.is_mac_os(): - # brew install ccache - utils.run_command(['brew', 'install', 'ccache']) - - # Install clang-format on linux/mac if its not installed already - if not utils.is_command_installed('clang-format'): - if utils.is_linux_os(): - # sudo apt install clang-format - utils.run_command(['apt', 'install', '-y','clang-format'], as_root=True) - elif utils.is_mac_os(): - # brew install protobuf - utils.run_command(['brew', 'install', 'clang-format']) - - # Install required python dependencies. - # On Catalina, python2 in installed as default python. - # Example command: - # python3 -m pip install -r external/pip_requirements.txt --user - utils.run_command( - ['python3' if utils.is_command_installed('python3') else 'python', '-m', - 'pip', 'install', '-r', 'external/pip_requirements.txt', '--user'] ) + args = parse_cmdline_args() + if not args.running_only: + # Install protobuf on linux/mac if its not installed already + if not utils.is_command_installed('protoc'): + if utils.is_linux_os(): + # sudo apt install protobuf-compiler + utils.run_command(['apt', 'install', '-y','protobuf-compiler'], as_root=True) + elif utils.is_mac_os(): + # brew install protobuf + utils.run_command(['brew', 'install', 'protobuf']) + + # Install go on linux/mac if its not installed already + if not utils.is_command_installed('go'): + if utils.is_linux_os(): + # sudo apt install -y golang + utils.run_command(['apt', 'install', '-y','golang'], as_root=True) + elif utils.is_mac_os(): + # brew install protobuf + utils.run_command(['brew', 'install', 'go']) + + # Install openssl on linux/mac if its not installed already + if not utils.is_command_installed('go'): + if utils.is_linux_os(): + # sudo apt install -y openssl + utils.run_command(['apt', 'install', '-y','openssl'], as_root=True) + elif utils.is_mac_os(): + # brew install protobuf + utils.run_command(['brew', 'install', 'openssl']) + + # Install ccache on linux/mac if its not installed already + if not utils.is_command_installed('ccache'): + if utils.is_linux_os(): + # sudo apt install ccache + utils.run_command(['apt', 'install', '-y', 'ccache'], as_root=True) + elif utils.is_mac_os(): + # brew install ccache + utils.run_command(['brew', 'install', 'ccache']) + + # Install clang-format on linux/mac if its not installed already + if not utils.is_command_installed('clang-format'): + if utils.is_linux_os(): + # sudo apt install clang-format + utils.run_command(['apt', 'install', '-y','clang-format'], as_root=True) + elif utils.is_mac_os(): + # brew install protobuf + utils.run_command(['brew', 'install', 'clang-format']) + + # Install required python dependencies. + # On Catalina, python2 in installed as default python. + # Example command: + # python3 -m pip install -r external/pip_requirements.txt --user + utils.run_command( + ['python3' if utils.is_command_installed('python3') else 'python', '-m', + 'pip', 'install', '-r', 'external/pip_requirements.txt', '--user'] ) + + if args.arch == 'x86': + utils.install_x86_support_libraries(args.gha_build) + +def parse_cmdline_args(): + parser = argparse.ArgumentParser(description='Install prerequisites for building cpp sdk') + parser.add_argument('--arch', default=None, help='Install support libraries to build a specific architecture (currently supported: x86)') + parser.add_argument('--running_only', action='store_true', help='Only install prerequisites for running, not for building') + parser.add_argument('--gha_build', action='store_true', default=None, help='Set to true when building on GitHub, changing some prerequisite installation behavior.') + args = parser.parse_args() + return args if __name__ == '__main__': main() From 34b87f9219799945d1d71579a891d87e3dda60ed Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Mon, 9 May 2022 12:09:42 -0700 Subject: [PATCH 83/88] Fix installation of openssl --- scripts/gha/install_prereqs_desktop.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/gha/install_prereqs_desktop.py b/scripts/gha/install_prereqs_desktop.py index 0cf4f54a5..c6bd8283f 100644 --- a/scripts/gha/install_prereqs_desktop.py +++ b/scripts/gha/install_prereqs_desktop.py @@ -51,16 +51,16 @@ def main(): # sudo apt install -y golang utils.run_command(['apt', 'install', '-y','golang'], as_root=True) elif utils.is_mac_os(): - # brew install protobuf + # brew install go utils.run_command(['brew', 'install', 'go']) # Install openssl on linux/mac if its not installed already - if not utils.is_command_installed('go'): + if not utils.is_command_installed('openssl'): if utils.is_linux_os(): # sudo apt install -y openssl utils.run_command(['apt', 'install', '-y','openssl'], as_root=True) elif utils.is_mac_os(): - # brew install protobuf + # brew install openssl utils.run_command(['brew', 'install', 'openssl']) # Install ccache on linux/mac if its not installed already From 23f9771f3ec3a270ef49d15200e50792f70367c2 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Mon, 9 May 2022 12:22:51 -0700 Subject: [PATCH 84/88] Add option to prereqs script to skip openssl, for boringssl builds. --- .github/workflows/cpp-packaging.yml | 4 ++-- .github/workflows/integration_tests.yml | 2 +- scripts/gha/install_prereqs_desktop.py | 5 +++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cpp-packaging.yml b/.github/workflows/cpp-packaging.yml index de845ba02..9cc495e8a 100644 --- a/.github/workflows/cpp-packaging.yml +++ b/.github/workflows/cpp-packaging.yml @@ -414,7 +414,7 @@ jobs: timeout_minutes: 15 max_attempts: 3 command: | - python scripts/gha/install_prereqs_desktop.py --gha_build --arch '${{ matrix.architecture }}' + python scripts/gha/install_prereqs_desktop.py --gha_build --arch '${{ matrix.architecture }}' --ssl boringssl - name: Export verbose flag shell: bash @@ -542,7 +542,7 @@ jobs: - name: Install prerequisites run: | cd sdk-src - python scripts/gha/install_prereqs_desktop.py + python scripts/gha/install_prereqs_desktop.py --ssl boringssl cd .. - name: postprocess and package built SDK diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 787031e30..3c0923e6f 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -277,7 +277,7 @@ jobs: timeout_minutes: 2 max_attempts: 3 command: | - python scripts/gha/install_prereqs_desktop.py + python scripts/gha/install_prereqs_desktop.py --ssl '${{ matrix.ssl_variant }}' pip install -r scripts/gha/requirements.txt python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}" - name: Install OpenSSL (Windows) diff --git a/scripts/gha/install_prereqs_desktop.py b/scripts/gha/install_prereqs_desktop.py index c6bd8283f..b32c82c44 100644 --- a/scripts/gha/install_prereqs_desktop.py +++ b/scripts/gha/install_prereqs_desktop.py @@ -55,7 +55,7 @@ def main(): utils.run_command(['brew', 'install', 'go']) # Install openssl on linux/mac if its not installed already - if not utils.is_command_installed('openssl'): + if not utils.is_command_installed('openssl') and args.ssl == 'openssl': if utils.is_linux_os(): # sudo apt install -y openssl utils.run_command(['apt', 'install', '-y','openssl'], as_root=True) @@ -96,7 +96,8 @@ def parse_cmdline_args(): parser = argparse.ArgumentParser(description='Install prerequisites for building cpp sdk') parser.add_argument('--arch', default=None, help='Install support libraries to build a specific architecture (currently supported: x86)') parser.add_argument('--running_only', action='store_true', help='Only install prerequisites for running, not for building') - parser.add_argument('--gha_build', action='store_true', default=None, help='Set to true when building on GitHub, changing some prerequisite installation behavior.') + parser.add_argument('--gha_build', action='store_true', default=None, help='Set this option when building on GitHub, changing some prerequisite installation behavior') + parser.add_argument('--ssl', default='openssl', help='Which SSL is this build using (supported: openssl, boringssl)') args = parser.parse_args() return args From b114ed239fb159539591ca2dad02a50fe4b41aae Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Mon, 9 May 2022 12:25:06 -0700 Subject: [PATCH 85/88] Fix conditional --- scripts/gha/install_prereqs_desktop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gha/install_prereqs_desktop.py b/scripts/gha/install_prereqs_desktop.py index b32c82c44..49452be9b 100644 --- a/scripts/gha/install_prereqs_desktop.py +++ b/scripts/gha/install_prereqs_desktop.py @@ -55,7 +55,7 @@ def main(): utils.run_command(['brew', 'install', 'go']) # Install openssl on linux/mac if its not installed already - if not utils.is_command_installed('openssl') and args.ssl == 'openssl': + if args.ssl == 'openssl' and not utils.is_command_installed('openssl'): if utils.is_linux_os(): # sudo apt install -y openssl utils.run_command(['apt', 'install', '-y','openssl'], as_root=True) From fdd1bedeceb9db719ca0b84d8f8149679b866586 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Mon, 9 May 2022 12:39:40 -0700 Subject: [PATCH 86/88] Use 'with' syntax for file i/o --- scripts/gha/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/gha/utils.py b/scripts/gha/utils.py index 7f964e414..3e4b8c18b 100644 --- a/scripts/gha/utils.py +++ b/scripts/gha/utils.py @@ -241,9 +241,9 @@ def install_x86_support_libraries(gha_build=False): packages = ['--allow-downgrades'] + packages + ['libpcre2-8-0=10.34-7'] # First check if these packages exist on the machine already - devnull = open(os.devnull, "w") - process = subprocess.run(["dpkg", "-s"] + packages, stdout=devnull, stderr=subprocess.STDOUT) - devnull.close() + with open(os.devnull, "w") as devnull: + process = subprocess.run(["dpkg", "-s"] + packages, stdout=devnull, stderr=subprocess.STDOUT) + if process.returncode != 0: # This implies not all of the required packages are already installed on user's machine # Install them. From 1b75bca3ace1db80f15319977a89c786a297c59c Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Mon, 9 May 2022 15:15:05 -0700 Subject: [PATCH 87/88] Remove space --- scripts/gha/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/gha/utils.py b/scripts/gha/utils.py index ee64edd55..b1ef778bc 100644 --- a/scripts/gha/utils.py +++ b/scripts/gha/utils.py @@ -260,4 +260,3 @@ def install_x86_support_libraries(gha_build=False): # Note: "-f" = "fix" - let apt do what it needs to do to fix dependencies. run_command(['apt', 'install', '-f', '-V', '-y', 'libsecret-1-dev'], as_root=True, check=True) - From 01a179a8118d643e094be495314b68b0ab0e0c2a Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Mon, 9 May 2022 15:33:40 -0700 Subject: [PATCH 88/88] Remove redundant matrix exclusion --- .github/workflows/integration_tests.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index f21b4d99b..4d5e33e00 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -849,9 +849,6 @@ jobs: - os: windows-latest ssl_variant: openssl arch: x86 - - os: macos-12 - ssl_variant: openssl - arch: arm64 # Custom for this matrix: MacOS GitHub-hosted runner cannot test arm64 # code. Exclude that scenario from running here; it will run in # test_desktop_custom_runners.