From f25df67c29e4e6e7c38c51ff31c42ff651b49ec5 Mon Sep 17 00:00:00 2001 From: "drsanta@google.com" Date: Mon, 14 Jun 2021 16:26:06 -0400 Subject: [PATCH 1/4] move ta artifacts to c for windows uploads --- .github/workflows/integration_tests.yml | 70 +++++++++++++++++++------ 1 file changed, 53 insertions(+), 17 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index c0e7827feb..2c39003d72 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -253,20 +253,20 @@ jobs: with: ref: ${{needs.prepare_matrix.outputs.github_ref}} submodules: true - - name: Set env vars (ubuntu) + - name: Set env vars (Linux) if: startsWith(matrix.os, 'ubuntu') run: echo "VCPKG_TRIPLET=x64-linux" >> $GITHUB_ENV - - name: Set env vars (macos) + - name: Set env vars (macOS) if: startsWith(matrix.os, 'macos') run: echo "VCPKG_TRIPLET=x64-osx" >> $GITHUB_ENV - - name: Set env vars (windows) + - name: Set env vars (Windows) shell: bash if: startsWith(matrix.os, 'windows') run: echo "VCPKG_TRIPLET=x64-windows-static" >> $GITHUB_ENV - - name: Set env vars(all) + - name: Set env vars (all) shell: bash run: echo "VCPKG_RESPONSE_FILE=external/vcpkg_${{ env.VCPKG_TRIPLET }}_response_file.txt" >> $GITHUB_ENV - - name: Add msbuild to PATH (windows) + - name: Add msbuild to PATH (Windows) if: startsWith(matrix.os, 'windows') uses: microsoft/setup-msbuild@v1.0.2 - name: Cache vcpkg C++ dependencies @@ -293,12 +293,12 @@ jobs: run: | pip install -r scripts/gha/requirements.txt python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}" - - name: Install OpenSSL (windows) + - name: Install OpenSSL (Windows) if: matrix.ssl_variant == 'openssl' && startsWith(matrix.os, 'windows') run: | choco install openssl -r - - name: Install OpenSSL (MacOS) + - name: Install OpenSSL (macOS) if: matrix.ssl_variant == 'openssl' && startsWith(matrix.os, 'macos') run: | @@ -341,7 +341,7 @@ jobs: --artifact_name "desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}" \ --noadd_timestamp \ --short_output_paths \ - ${additional_flags[*]} + ${additional_flags[*]} - name: Upload Desktop integration tests artifact uses: actions/upload-artifact@v2.2.2 if: ${{ !cancelled() }} @@ -356,6 +356,38 @@ jobs: name: log-artifact path: build-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}* retention-days: ${{ env.artifactRetentionDays }} + - name: Cleanup Local Copies of Uploaded Artifacts + shell: bash + run: | + rm -rf testapps-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }} + - name: Configure GCS Upload Directory (Linux, macOS) + if: (startsWith(matrix.os, 'ubuntu') || startsWith(matrix.os, 'macos')) && !cancelled() + shell: bash + run: echo "GCS_UPLOAD_DIR=${{ github.workspace }}" >> $GITHUB_ENV + - name: Configure GCS Upload Directory (Windows) + # For Windows move the uploads to C: to create enough space for the + # Cloud SDK installation on D:. + if: startsWith(matrix.os, 'windows') && !cancelled() + shell: bash + run: echo "GCS_UPLOAD_DIR=/c/fpl-upload" >> $GITHUB_ENV + - name: Move GCS Artifacts to GCS Upload Dir + # Copy the artifacts and also the upload scripts since python's relpath + # produces errors when called across two different drives. Use tar to + # move the files as it neatly deepcopies firestore's symlinks. + if: ( ${{ env.GCS_UPLOAD_DIR != github.workspace }} ) && !cancelled() + shell: bash + run: | + set -x + mkdir -p ${{env.GCS_UPLOAD_DIR}}/scripts/gha/integration_testing + mkdir -p ${{env.GCS_UPLOAD_DIR}}/scripts/gha-encrypted + tar -chf ${{env.GCS_UPLOAD_DIR}}/ta.tar ta + rm -rf ta + tar xf ${{env.GCS_UPLOAD_DIR}}/ta.tar -C ${{env.GCS_UPLOAD_DIR}} + rm ${{env.GCS_UPLOAD_DIR}}/ta.tar + cp scripts/gha/gcs_uploader.py ${{env.GCS_UPLOAD_DIR}}/scripts/gha-encrypted/ + cp scripts/gha/integration_testing/gcs.py ${{env.GCS_UPLOAD_DIR}}/scripts/gha/integration_testing/ + cp scripts/gha/gcs_uploader.py ${{env.GCS_UPLOAD_DIR}}/scripts/gha/ + cp scripts/gha-encrypted/gcs_key_file.json ${{env.GCS_UPLOAD_DIR}}//scripts/gha-encrypted/ - name: Set CLOUDSDK_PYTHON (Windows) shell: bash if: startsWith(matrix.os, 'windows') && !cancelled() @@ -364,8 +396,12 @@ jobs: if: ${{ !cancelled() }} uses: google-github-actions/setup-gcloud@master - name: Upload Desktop Artifacts to GCS + shell: bash if: ${{ !cancelled() }} - run: python scripts/gha/gcs_uploader.py --testapp_dir ta --key_file scripts/gha-encrypted/gcs_key_file.json + run: | + pushd ${{env.GCS_UPLOAD_DIR}} + python scripts/gha/gcs_uploader.py --testapp_dir ta --key_file scripts/gha-encrypted/gcs_key_file.json + popd - name: Add failure label # We can mark a failure as soon as any one test fails. if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} @@ -431,20 +467,20 @@ jobs: with: ref: ${{needs.prepare_matrix.outputs.github_ref}} submodules: true - - name: Set env vars (ubuntu) + - name: Set env vars (Linux) if: startsWith(matrix.os, 'ubuntu') run: echo "VCPKG_TRIPLET=x64-linux" >> $GITHUB_ENV - - name: Set env vars (macos) + - name: Set env vars (macOS) if: startsWith(matrix.os, 'macos') run: echo "VCPKG_TRIPLET=x64-osx" >> $GITHUB_ENV - - name: Set env vars (windows) + - name: Set env vars (Windows) shell: bash - if: startsWith(matrix.os, 'windows') + if: startsWith(matrix.os, 'Windows') run: echo "VCPKG_TRIPLET=x64-windows-static" >> $GITHUB_ENV - - name: Set env vars(all) + - name: Set env vars (all) shell: bash run: echo "VCPKG_RESPONSE_FILE=external/vcpkg_${{ env.VCPKG_TRIPLET }}_response_file.txt" >> $GITHUB_ENV - - name: Add msbuild to PATH (windows) + - name: Add msbuild to PATH (Windows) if: startsWith(matrix.os, 'windows') uses: microsoft/setup-msbuild@v1.0.2 - name: Cache NDK @@ -575,9 +611,9 @@ jobs: with: ref: ${{needs.prepare_matrix.outputs.github_ref}} submodules: true - - name: Set env vars (macos) + - name: Set env vars (macOS) run: echo "VCPKG_TRIPLET=x64-osx" >> $GITHUB_ENV - - name: Set env vars(all) + - name: Set env vars (all) shell: bash run: echo "VCPKG_RESPONSE_FILE=external/vcpkg_${{ env.VCPKG_TRIPLET }}_response_file.txt" >> $GITHUB_ENV - name: Setup python From 7308aa7172171985cbe84c0273062c8dad582f58 Mon Sep 17 00:00:00 2001 From: "drsanta@google.com" Date: Mon, 14 Jun 2021 18:00:42 -0400 Subject: [PATCH 2/4] attempt to rework cancelled clause --- .github/workflows/integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 2c39003d72..c7ab68a36c 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -374,7 +374,7 @@ jobs: # Copy the artifacts and also the upload scripts since python's relpath # produces errors when called across two different drives. Use tar to # move the files as it neatly deepcopies firestore's symlinks. - if: ( ${{ env.GCS_UPLOAD_DIR != github.workspace }} ) && !cancelled() + if: ${{ (env.GCS_UPLOAD_DIR != github.workspace) && !canceled() }} shell: bash run: | set -x From 4a69666e86dd6ba8c72155e8a65e12cb9c6f4f21 Mon Sep 17 00:00:00 2001 From: "drsanta@google.com" Date: Mon, 14 Jun 2021 18:02:19 -0400 Subject: [PATCH 3/4] cancelled --- .github/workflows/integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index c7ab68a36c..aad01b9f2a 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -374,7 +374,7 @@ jobs: # Copy the artifacts and also the upload scripts since python's relpath # produces errors when called across two different drives. Use tar to # move the files as it neatly deepcopies firestore's symlinks. - if: ${{ (env.GCS_UPLOAD_DIR != github.workspace) && !canceled() }} + if: ${{ (env.GCS_UPLOAD_DIR != github.workspace) && !cancelled() }} shell: bash run: | set -x From 02c695056c8e6b60750499f81bc8ba2608faeef0 Mon Sep 17 00:00:00 2001 From: "drsanta@google.com" Date: Tue, 15 Jun 2021 16:02:32 -0400 Subject: [PATCH 4/4] comment describing the ta directory contents --- .github/workflows/integration_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index aad01b9f2a..14c01779f4 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -380,6 +380,7 @@ jobs: set -x mkdir -p ${{env.GCS_UPLOAD_DIR}}/scripts/gha/integration_testing mkdir -p ${{env.GCS_UPLOAD_DIR}}/scripts/gha-encrypted + # The ta directory contains the integration testapps to archive. tar -chf ${{env.GCS_UPLOAD_DIR}}/ta.tar ta rm -rf ta tar xf ${{env.GCS_UPLOAD_DIR}}/ta.tar -C ${{env.GCS_UPLOAD_DIR}}