diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 14c01779f4..783c8c060a 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -4,7 +4,7 @@ on: schedule: - cron: "0 9 * * *" # 9am UTC = 1am PST / 2am PDT pull_request: - types: [ labeled ] + types: [ labeled, closed ] workflow_dispatch: inputs: @@ -50,26 +50,26 @@ env: jobs: check_trigger: - ### This job only runs when the workflow was triggered by a PR getting labeled. - ### It checks whether the label is a test-request trigger (cancelling - ### the workflow if not), and then sets the in-progress label. It sets - ### outputs to control the build_* matrix (full or quick) and to tell + ### This job runs when the workflow was triggered by 1) PR getting labeled + ### 2) PR merged & closed 3) or manumlly triggered with Pull request #. + ### If triggered by label, then checks whether the label is a test-request + ### trigger (cancelling the workflow if not). + ### It sets outputs to control the build_* matrix (full or quick) and to tell ### subsequent steps to update the labels as well. runs-on: ubuntu-latest if: | - (github.event_name == 'pull_request' && github.event.action == 'labeled') || - (github.event.inputs.test_pull_request != '') + github.event_name == 'pull_request' || github.event.inputs.test_pull_request != '' outputs: - should_update_labels: ${{ steps.set_outputs.outputs.should_update_labels }} + should_update_pr: ${{ steps.set_outputs.outputs.should_update_pr }} requested_tests: ${{ steps.set_outputs.outputs.requested_tests }} pr_number: ${{ steps.get_pr_number.outputs.pr_number }} steps: - ### If the label isn't one of the test-request triggers, cancel the workflow. - - name: Cancel workflow if label is irrelevant - if: ${{ !startsWith(github.event.label.name, env.triggerLabelPrefix) && github.event.inputs.test_pull_request == '' }} + ### If the label isn't 1)the test-request triggers, 2) closed on merge trigger, cancel the workflow. + - name: Cancel workflow + if: (github.event.action == 'labeled' && !startsWith(github.event.label.name, env.triggerLabelPrefix)) || (github.event.action == 'closed' && github.event.pull_request.merged != true) uses: andymckay/cancel-action@0.2 - - name: Wait for above cancellation if label is irrelevant - if: ${{ !startsWith(github.event.label.name, env.triggerLabelPrefix) && github.event.inputs.test_pull_request == '' }} + - name: Wait for above cancellation + if: (github.event.action == 'labeled' && !startsWith(github.event.label.name, env.triggerLabelPrefix)) || (github.event.action == 'closed' && github.event.pull_request.merged != true) run: | sleep 300 exit 1 # fail out if the cancellation above somehow failed. @@ -89,9 +89,7 @@ jobs: echo "::set-output name=pr_number::${{ github.event.inputs.test_pull_request }}" echo "::set-output name=github_ref::refs/pull/${{github.event.inputs.test_pull_request}}/merge" fi - ### Below this line, the label is one of the test-request triggers. - name: Cancel previous runs on the same PR - if: github.event_name == 'pull_request' uses: styfle/cancel-workflow-action@0.8.0 with: access_token: ${{ github.token }} @@ -110,7 +108,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} - id: set_outputs run: | - echo "::set-output name=should_update_labels::1" + echo "::set-output name=should_update_pr::1" if [[ "${{ github.event.label.name }}" == "${{ env.triggerLabelFull }}" ]]; then echo "::set-output name=requested_tests::full" elif [[ "${{ github.event.label.name }}" == "${{ env.triggerLabelQuick }}" ]]; then @@ -120,55 +118,23 @@ jobs: echo "::set-output name=requested_tests::auto" fi ### Add the in-progress label and remove any previous success/fail labels. - - name: Add in-progress label - uses: actions-ecosystem/action-add-labels@v1 + - uses: actions/checkout@v2 with: - github_token: ${{ github.token }} - number: "${{ steps.get_pr_number.outputs.pr_number }}" - labels: "${{ env.statusLabelInProgress }}" - - name: Remove previous succeeded label - uses: actions-ecosystem/action-remove-labels@v1 - with: - github_token: ${{ github.token }} - number: "${{ steps.get_pr_number.outputs.pr_number }}" - labels: "${{ env.statusLabelSucceeded }}" - - name: Remove previous failed label - uses: actions-ecosystem/action-remove-labels@v1 - with: - github_token: ${{ github.token }} - number: "${{ steps.get_pr_number.outputs.pr_number }}" - labels: "${{ env.statusLabelFailed }}" - - name: Find old status comment, if any - uses: peter-evans/find-comment@v1 - id: find-comment - with: - issue-number: ${{steps.get_pr_number.outputs.pr_number}} - body-includes: ${{ env.statusCommentIdentifier }} - token: ${{github.token}} - - name: Delete old status comment - if: ${{ steps.find-comment.outputs.comment-id != 0 }} - uses: jungwinter/comment@v1 + ref: ${{steps.get_pr_number.outputs.github_ref}} + - name: Setup python + uses: actions/setup-python@v2 with: - type: delete - comment_id: ${{ steps.find-comment.outputs.comment-id }} - token: ${{ github.token }} - - name: Get current time for status comment - id: get-time - shell: bash + python-version: ${{ env.pythonVersion }} + - name: Install python deps + run: pip install -r scripts/gha/requirements.txt + - name: Update PR label and comment run: | - echo -n "::set-output name=time::" - TZ=America/Los_Angeles date - - name: Add in progress status comment - uses: phulsechinmay/rewritable-pr-comment@v0.3.0 - with: - message: | - ### ⏳  Integration test in progress... - Requested by @${{github.actor}} on commit ${{steps.get_pr_number.outputs.github_ref}} - Last updated: ${{ steps.get-time.outputs.time }} - **[View integration test run](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}})** - GITHUB_TOKEN: ${{ github.token }} - COMMENT_IDENTIFIER: ${{ env.statusCommentIdentifier }} - ISSUE_ID: ${{steps.get_pr_number.outputs.pr_number}} + python scripts/gha/it_workflow.py --stage start \ + --token ${{github.token}} \ + --issue_number ${{steps.get_pr_number.outputs.pr_number}} \ + --actor ${{github.actor}} \ + --commit ${{steps.get_pr_number.outputs.github_ref}} \ + --run_id ${{github.run_id}} # To feed input into the job matrix, we first need to convert to a JSON # list. Then we can use fromJson to define the field in the matrix for the tests job. @@ -403,48 +369,23 @@ jobs: pushd ${{env.GCS_UPLOAD_DIR}} python scripts/gha/gcs_uploader.py --testapp_dir ta --key_file scripts/gha-encrypted/gcs_key_file.json popd - - name: Add failure label - # We can mark a failure as soon as any one test fails. - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} - uses: actions-ecosystem/action-add-labels@v1 - with: - github_token: ${{ github.token }} - number: "${{needs.check_trigger.outputs.pr_number}}" - labels: "${{ env.statusLabelFailed }}" - - name: Get current time for status comment - id: get-time - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} - shell: bash - run: | - echo -n "::set-output name=time::" - TZ=America/Los_Angeles date - name: Download log artifacts + if: ${{ needs.check_trigger.outputs.should_update_pr && failure() && !cancelled() }} uses: actions/download-artifact@v2.0.8 - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} with: path: test_results name: log-artifact - - name: Get summary of test results - id: get-summary + - name: Update PR label and comment shell: bash - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} + if: ${{ needs.check_trigger.outputs.should_update_pr && failure() && !cancelled() }} run: | - echo 'SUMMARY_TABLE<> $GITHUB_ENV - python scripts/gha/summarize_test_results.py --dir test_results --markdown >> $GITHUB_ENV - echo 'EOF' >> $GITHUB_ENV - - name: Add failure status comment - uses: phulsechinmay/rewritable-pr-comment@v0.3.0 - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} - with: - message: | - ### ❌  Integration test FAILED (but still ⏳  in progress) - Requested by @${{github.actor}} on commit ${{needs.prepare_matrix.outputs.github_ref}} - Last updated: ${{ steps.get-time.outputs.time }} - **[View integration test results](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}})** - ${{ env.SUMMARY_TABLE }} - GITHUB_TOKEN: ${{ github.token }} - COMMENT_IDENTIFIER: ${{ env.statusCommentIdentifier }} - ISSUE_ID: ${{needs.check_trigger.outputs.pr_number}} + pushd ${{env.GCS_UPLOAD_DIR}} + python scripts/gha/it_workflow.py --stage progress \ + --token ${{github.token}} \ + --issue_number ${{needs.check_trigger.outputs.pr_number}}\ + --actor ${{github.actor}} \ + --commit ${{needs.prepare_matrix.outputs.github_ref}} \ + --run_id ${{github.run_id}} - name: Summarize build results if: ${{ !cancelled() }} shell: bash @@ -549,48 +490,21 @@ jobs: name: log-artifact path: build-results-android-${{ matrix.os }}* retention-days: ${{ env.artifactRetentionDays }} - - name: Add failure label - # We can mark a failure as soon as any one test fails. - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} - uses: actions-ecosystem/action-add-labels@v1 - with: - github_token: ${{ github.token }} - number: "${{needs.check_trigger.outputs.pr_number}}" - labels: "${{ env.statusLabelFailed }}" - - name: Get current time for status comment - id: get-time - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} - shell: bash - run: | - echo -n "::set-output name=time::" - TZ=America/Los_Angeles date - name: Download log artifacts + if: ${{ needs.check_trigger.outputs.should_update_pr && failure() && !cancelled() }} uses: actions/download-artifact@v2.0.8 - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} with: path: test_results name: log-artifact - - name: Get summary of test results - id: get-summary - shell: bash - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} - run: | - echo 'SUMMARY_TABLE<> $GITHUB_ENV - python scripts/gha/summarize_test_results.py --dir test_results --markdown >> $GITHUB_ENV - echo 'EOF' >> $GITHUB_ENV - - name: Add failure status comment - uses: phulsechinmay/rewritable-pr-comment@v0.3.0 - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} - with: - message: | - ### ❌  Integration test FAILED (but still ⏳  in progress) - Requested by @${{github.actor}} on commit ${{needs.prepare_matrix.outputs.github_ref}} - Last updated: ${{ steps.get-time.outputs.time }} - **[View integration test results](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}})** - ${{ env.SUMMARY_TABLE }} - GITHUB_TOKEN: ${{ github.token }} - COMMENT_IDENTIFIER: ${{ env.statusCommentIdentifier }} - ISSUE_ID: ${{needs.check_trigger.outputs.pr_number}} + - name: Update PR label and comment + if: ${{ needs.check_trigger.outputs.should_update_pr && failure() && !cancelled() }} + run: | + python scripts/gha/it_workflow.py --stage progress \ + --token ${{github.token}} \ + --issue_number ${{needs.check_trigger.outputs.pr_number}}\ + --actor ${{github.actor}} \ + --commit ${{needs.prepare_matrix.outputs.github_ref}} \ + --run_id ${{github.run_id}} - name: Summarize build results if: ${{ !cancelled() }} shell: bash @@ -674,48 +588,21 @@ jobs: name: log-artifact path: build-results-ios-macos-latest* retention-days: ${{ env.artifactRetentionDays }} - - name: Add failure label - # We can mark a failure as soon as any one test fails. - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} - uses: actions-ecosystem/action-add-labels@v1 - with: - github_token: ${{ github.token }} - number: "${{needs.check_trigger.outputs.pr_number}}" - labels: "${{ env.statusLabelFailed }}" - - name: Get current time for status comment - id: get-time - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} - shell: bash - run: | - echo -n "::set-output name=time::" - TZ=America/Los_Angeles date - name: Download log artifacts + if: ${{ needs.check_trigger.outputs.should_update_pr && failure() && !cancelled() }} uses: actions/download-artifact@v2.0.8 - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} with: path: test_results name: log-artifact - - name: Get summary of test results - id: get-summary - shell: bash - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} - run: | - echo 'SUMMARY_TABLE<> $GITHUB_ENV - python scripts/gha/summarize_test_results.py --dir test_results --markdown >> $GITHUB_ENV - echo 'EOF' >> $GITHUB_ENV - - name: Add failure status comment - uses: phulsechinmay/rewritable-pr-comment@v0.3.0 - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} - with: - message: | - ### ❌  Integration test FAILED (but still ⏳  in progress) - Requested by @${{github.actor}} on commit ${{needs.prepare_matrix.outputs.github_ref}} - Last updated: ${{ steps.get-time.outputs.time }} - **[View integration test results](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}})** - ${{ env.SUMMARY_TABLE }} - GITHUB_TOKEN: ${{ github.token }} - COMMENT_IDENTIFIER: ${{ env.statusCommentIdentifier }} - ISSUE_ID: ${{needs.check_trigger.outputs.pr_number}} + - name: Update PR label and comment + if: ${{ needs.check_trigger.outputs.should_update_pr && failure() && !cancelled() }} + run: | + python scripts/gha/it_workflow.py --stage progress \ + --token ${{github.token}} \ + --issue_number ${{needs.check_trigger.outputs.pr_number}}\ + --actor ${{github.actor}} \ + --commit ${{needs.prepare_matrix.outputs.github_ref}} \ + --run_id ${{github.run_id}} - name: Summarize build results if: ${{ !cancelled() }} shell: bash @@ -760,49 +647,22 @@ jobs: with: name: log-artifact path: testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}* - retention-days: ${{ env.artifactRetentionDays }} - - name: Add failure label - # We can mark a failure as soon as any one test fails. - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} - uses: actions-ecosystem/action-add-labels@v1 - with: - github_token: ${{ github.token }} - number: "${{needs.check_trigger.outputs.pr_number}}" - labels: "${{ env.statusLabelFailed }}" - - name: Get current time for status comment - id: get-time - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} - shell: bash - run: | - echo -n "::set-output name=time::" - TZ=America/Los_Angeles date + retention-days: ${{ env.artifactRetentionDays }} - name: Download log artifacts + if: ${{ needs.check_trigger.outputs.should_update_pr && failure() && !cancelled() }} uses: actions/download-artifact@v2.0.8 - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} with: path: test_results name: log-artifact - - name: Get summary of test results - id: get-summary - shell: bash - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} - run: | - echo 'SUMMARY_TABLE<> $GITHUB_ENV - python scripts/gha/summarize_test_results.py --dir test_results --markdown >> $GITHUB_ENV - echo 'EOF' >> $GITHUB_ENV - - name: Add failure status comment - uses: phulsechinmay/rewritable-pr-comment@v0.3.0 - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} - with: - message: | - ### ❌  Integration test FAILED (but still ⏳  in progress) - Requested by @${{github.actor}} on commit ${{needs.prepare_matrix.outputs.github_ref}} - Last updated: ${{ steps.get-time.outputs.time }} - **[View integration test results](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}})** - ${{ env.SUMMARY_TABLE }} - GITHUB_TOKEN: ${{ github.token }} - COMMENT_IDENTIFIER: ${{ env.statusCommentIdentifier }} - ISSUE_ID: ${{needs.check_trigger.outputs.pr_number}} + - name: Update PR label and comment + if: ${{ needs.check_trigger.outputs.should_update_pr && failure() && !cancelled() }} + run: | + python scripts/gha/it_workflow.py --stage progress \ + --token ${{github.token}} \ + --issue_number ${{needs.check_trigger.outputs.pr_number}}\ + --actor ${{github.actor}} \ + --commit ${{needs.prepare_matrix.outputs.github_ref}} \ + --run_id ${{github.run_id}} - name: Summarize test results if: ${{ !cancelled() }} shell: bash @@ -865,49 +725,22 @@ jobs: with: name: log-artifact path: testapps/test-results-android-${{ matrix.build_os }}-${{ matrix.android_device }}* - retention-days: ${{ env.artifactRetentionDays }} - - name: Add failure label - # We can mark a failure as soon as any one test fails. - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} - uses: actions-ecosystem/action-add-labels@v1 - with: - github_token: ${{ github.token }} - number: "${{needs.check_trigger.outputs.pr_number}}" - labels: "${{ env.statusLabelFailed }}" - - name: Get current time for status comment - id: get-time - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} - shell: bash - run: | - echo -n "::set-output name=time::" - TZ=America/Los_Angeles date + retention-days: ${{ env.artifactRetentionDays }} - name: Download log artifacts + if: ${{ needs.check_trigger.outputs.should_update_pr && failure() && !cancelled() }} uses: actions/download-artifact@v2.0.8 - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} with: path: test_results name: log-artifact - - name: Get summary of test results - id: get-summary - shell: bash - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} - run: | - echo 'SUMMARY_TABLE<> $GITHUB_ENV - python scripts/gha/summarize_test_results.py --dir test_results --markdown >> $GITHUB_ENV - echo 'EOF' >> $GITHUB_ENV - - name: Add failure status comment - uses: phulsechinmay/rewritable-pr-comment@v0.3.0 - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} - with: - message: | - ### ❌  Integration test FAILED (but still ⏳  in progress) - Requested by @${{github.actor}} on commit ${{needs.prepare_matrix.outputs.github_ref}} - Last updated: ${{ steps.get-time.outputs.time }} - **[View integration test results](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}})** - ${{ env.SUMMARY_TABLE }} - GITHUB_TOKEN: ${{ github.token }} - COMMENT_IDENTIFIER: ${{ env.statusCommentIdentifier }} - ISSUE_ID: ${{needs.check_trigger.outputs.pr_number}} + - name: Update PR label and comment + if: ${{ needs.check_trigger.outputs.should_update_pr && failure() && !cancelled() }} + run: | + python scripts/gha/it_workflow.py --stage progress \ + --token ${{github.token}} \ + --issue_number ${{needs.check_trigger.outputs.pr_number}}\ + --actor ${{github.actor}} \ + --commit ${{needs.prepare_matrix.outputs.github_ref}} \ + --run_id ${{github.run_id}} - name: Summarize test results if: ${{ !cancelled() }} shell: bash @@ -963,55 +796,28 @@ jobs: --logfile_name "ios-macos-latest-${{ matrix.ios_device }}" \ --code_platform cpp \ --key_file scripts/gha-encrypted/gcs_key_file.json - - name: Upload Android test results artifact + - name: Upload iOS test results artifact if: ${{ !cancelled() }} uses: actions/upload-artifact@v2.2.2 with: name: log-artifact path: testapps/test-results-ios-macos-latest-${{ matrix.ios_device }}* - retention-days: ${{ env.artifactRetentionDays }} - - name: Add failure label - # We can mark a failure as soon as any one test fails. - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} - uses: actions-ecosystem/action-add-labels@v1 - with: - github_token: ${{ github.token }} - number: "${{needs.check_trigger.outputs.pr_number}}" - labels: "${{ env.statusLabelFailed }}" - - name: Get current time for status comment - id: get-time - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} - shell: bash - run: | - echo -n "::set-output name=time::" - TZ=America/Los_Angeles date + retention-days: ${{ env.artifactRetentionDays }} - name: Download log artifacts + if: ${{ needs.check_trigger.outputs.should_update_pr && failure() && !cancelled() }} uses: actions/download-artifact@v2.0.8 - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} with: path: test_results name: log-artifact - - name: Get summary of test results - id: get-summary - shell: bash - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} - run: | - echo 'SUMMARY_TABLE<> $GITHUB_ENV - python scripts/gha/summarize_test_results.py --dir test_results --markdown >> $GITHUB_ENV - echo 'EOF' >> $GITHUB_ENV - - name: Add failure status comment - uses: phulsechinmay/rewritable-pr-comment@v0.3.0 - if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} - with: - message: | - ### ❌  Integration test FAILED (but still ⏳  in progress) - Requested by @${{github.actor}} on commit ${{needs.prepare_matrix.outputs.github_ref}} - Last updated: ${{ steps.get-time.outputs.time }} - **[View integration test results](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}})** - ${{ env.SUMMARY_TABLE }} - GITHUB_TOKEN: ${{ github.token }} - COMMENT_IDENTIFIER: ${{ env.statusCommentIdentifier }} - ISSUE_ID: ${{needs.check_trigger.outputs.pr_number}} + - name: Update PR label and comment + if: ${{ needs.check_trigger.outputs.should_update_pr && failure() && !cancelled() }} + run: | + python scripts/gha/it_workflow.py --stage progress \ + --token ${{github.token}} \ + --issue_number ${{needs.check_trigger.outputs.pr_number}}\ + --actor ${{github.actor}} \ + --commit ${{needs.prepare_matrix.outputs.github_ref}} \ + --run_id ${{github.run_id}} - name: Summarize test results if: ${{ !cancelled() }} shell: bash @@ -1020,55 +826,13 @@ jobs: if [[ "${{ job.status }}" != "success" ]]; then exit 1 fi - - add_success_label: - name: "add-success-label" - needs: [check_trigger, prepare_matrix, test_desktop, test_android, test_ios] - runs-on: ubuntu-latest - if: ${{ needs.check_trigger.outputs.should_update_labels && !cancelled() && !failure() }} - steps: - - name: Add success label - uses: actions-ecosystem/action-add-labels@v1 - with: - github_token: ${{ github.token }} - number: "${{needs.check_trigger.outputs.pr_number}}" - labels: "${{ env.statusLabelSucceeded }}" - - name: Get current time for status comment - id: get-time - shell: bash - run: | - echo -n "::set-output name=time::" - TZ=America/Los_Angeles date - - name: Add success status comment - uses: phulsechinmay/rewritable-pr-comment@v0.3.0 - with: - message: | - ### ✅  Integration test succeeded! - Requested by @${{github.actor}} on commit ${{needs.prepare_matrix.outputs.github_ref}} - Last updated: ${{ steps.get-time.outputs.time }} - **[View integration test results](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}})** - GITHUB_TOKEN: ${{ github.token }} - COMMENT_IDENTIFIER: ${{ env.statusCommentIdentifier }} - ISSUE_ID: ${{needs.check_trigger.outputs.pr_number}} - add_failure_label: - name: "add-failure-label" + summarize_results: + name: "summarize-results" needs: [check_trigger, prepare_matrix, test_desktop, test_android, test_ios] runs-on: ubuntu-latest - if: ${{ needs.check_trigger.outputs.should_update_labels && !cancelled() && failure() }} + if: ${{ !cancelled() }} steps: - - name: Add failure label - uses: actions-ecosystem/action-add-labels@v1 - with: - github_token: ${{ github.token }} - number: "${{needs.check_trigger.outputs.pr_number}}" - labels: "${{ env.statusLabelFailed }}" - - name: Get current time for status comment - id: get-time - shell: bash - run: | - echo -n "::set-output name=time::" - TZ=America/Los_Angeles date - uses: actions/checkout@v2 with: ref: ${{needs.prepare_matrix.outputs.github_ref}} @@ -1077,37 +841,12 @@ jobs: with: python-version: ${{ env.pythonVersion }} - name: Install python deps - run: python scripts/gha/install_prereqs_desktop.py + run: pip install -r scripts/gha/requirements.txt - name: Download log artifacts uses: actions/download-artifact@v2.0.8 with: path: test_results name: log-artifact - - name: Get summary of test results - shell: bash - run: | - echo 'SUMMARY_TABLE<> $GITHUB_ENV - python scripts/gha/summarize_test_results.py --dir test_results --markdown >> $GITHUB_ENV - echo 'EOF' >> $GITHUB_ENV - - name: Add failure status comment - uses: phulsechinmay/rewritable-pr-comment@v0.3.0 - with: - message: | - ### ❌  Integration test FAILED - Requested by @${{github.actor}} on commit ${{needs.prepare_matrix.outputs.github_ref}} - Last updated: ${{ steps.get-time.outputs.time }} - **[Download integration test detailed logs and artifacts](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}})** - ${{ env.SUMMARY_TABLE }} - GITHUB_TOKEN: ${{ github.token }} - COMMENT_IDENTIFIER: ${{ env.statusCommentIdentifier }} - ISSUE_ID: ${{needs.check_trigger.outputs.pr_number}} - - remove_in_progress_label: - name: "remove-in-progress-label" - needs: [check_trigger, prepare_matrix, test_desktop, test_android, test_ios] - runs-on: ubuntu-latest - if: ${{ needs.check_trigger.outputs.should_update_labels && !cancelled() }} - steps: # Use a different token to remove the "in-progress" label, # to allow the removal to trigger the "Check Labels" workflow. - name: Generate token for GitHub API @@ -1116,32 +855,23 @@ jobs: with: app_id: ${{ secrets.WORKFLOW_TRIGGER_APP_ID }} private_key: ${{ secrets.WORKFLOW_TRIGGER_APP_PRIVATE_KEY }} - - name: Remove in progress label - uses: actions-ecosystem/action-remove-labels@v1 - with: - github_token: ${{steps.generate-token.outputs.token}} - number: "${{needs.check_trigger.outputs.pr_number}}" - labels: "${{ env.statusLabelInProgress }}" - - summarize_results: - name: "summarize-results" - needs: [prepare_matrix, add_failure_label, add_success_label, remove_in_progress_label, test_desktop, test_android, test_ios] - runs-on: ubuntu-latest - if: ${{ !cancelled() }} - steps: - - uses: actions/checkout@v2 - with: - ref: ${{needs.prepare_matrix.outputs.github_ref}} - - name: Setup python - uses: actions/setup-python@v2 - with: - python-version: ${{ env.pythonVersion }} - - name: Install python deps - run: python scripts/gha/install_prereqs_desktop.py - - name: Download log artifacts - uses: actions/download-artifact@v2.0.8 - with: - path: test_results - name: log-artifact + - name: Update PR label and comment + if: ${{ needs.check_trigger.outputs.should_update_pr }} + run: | + python scripts/gha/it_workflow.py --stage end \ + --token ${{github.token}} \ + --issue_number ${{needs.check_trigger.outputs.pr_number}}\ + --actor ${{github.actor}} \ + --commit ${{needs.prepare_matrix.outputs.github_ref}} \ + --run_id ${{github.run_id}} \ + --new_token ${{steps.generate-token.outputs.token}} + - name: Update Daily Report + if: ${{ github.event_name == 'schedule' }} + run: | + python scripts/gha/it_workflow.py --stage report \ + --token ${{github.token}} \ + --actor ${{github.actor}} \ + --commit ${{needs.prepare_matrix.outputs.github_ref}} \ + --run_id ${{github.run_id}} - name: Summarize results into GitHub log run: python scripts/gha/summarize_test_results.py --dir test_results --github_log diff --git a/scripts/gha/build_testapps.py b/scripts/gha/build_testapps.py index 0ad5b51437..a3a34859df 100644 --- a/scripts/gha/build_testapps.py +++ b/scripts/gha/build_testapps.py @@ -74,24 +74,23 @@ """ import datetime -from distutils import dir_util import os import platform import shutil import subprocess import sys import json +import attr from absl import app from absl import flags from absl import logging +from distutils import dir_util -import attr - +import utils from integration_testing import config_reader from integration_testing import test_validation from integration_testing import xcodebuild -import utils # Environment variables _JAVA_HOME = "JAVA_HOME" diff --git a/scripts/gha/github.py b/scripts/gha/github.py new file mode 100644 index 0000000000..94c0ad702f --- /dev/null +++ b/scripts/gha/github.py @@ -0,0 +1,142 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A utility for GitHub REST API. + +This script handles GitHub Issue, Pull Request, Comment, Label and Artifact + +""" + +import requests +import json +import shutil + +from absl import logging + +OWNER = 'firebase' +REPO = 'firebase-cpp-sdk' + +BASE_URL = 'https://api.github.com' +FIREBASE_URL = '%s/repos/%s/%s' % (BASE_URL, OWNER, REPO) +logging.set_verbosity(logging.INFO) + +def create_issue(token, title, label): + """Create an issue: https://docs.github.com/en/rest/reference/issues#create-an-issue""" + url = f'{FIREBASE_URL}/issues' + headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'} + data = {'title': title, 'labels': [label]} + with requests.post(url, headers=headers, data=json.dumps(data)) as response: + logging.info("create_issue: %s response: %s", url, response) + return response.json() + + +def update_issue(token, issue_number, data): + """Update an issue: https://docs.github.com/en/rest/reference/issues#update-an-issue""" + url = f'{FIREBASE_URL}/issues/{issue_number}' + headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'} + with requests.patch(url, headers=headers, data=json.dumps(data)) as response: + logging.info("update_issue: %s response: %s", url, response) + + +def open_issue(token, issue_number): + update_issue(token, issue_number, data={'state': 'open'}) + + +def close_issue(token, issue_number): + update_issue(token, issue_number, data={'state': 'closed'}) + + +def update_issue_comment(token, issue_number, comment): + update_issue(token, issue_number, data={'body': comment}) + + +def search_issues_by_label(label): + """https://docs.github.com/en/rest/reference/search#search-issues-and-pull-requests""" + url = f'{BASE_URL}/search/issues?q=repo:{OWNER}/{REPO}+label:"{label}"+is:issue' + headers = {'Accept': 'application/vnd.github.v3+json'} + with requests.get(url, headers=headers) as response: + logging.info("search_issues_by_label: %s response: %s", url, response) + return response.json()["items"] + + +def list_comments(issue_number): + """https://docs.github.com/en/rest/reference/issues#list-issue-comments""" + url = f'{FIREBASE_URL}/issues/{issue_number}/comments' + headers = {'Accept': 'application/vnd.github.v3+json'} + with requests.get(url, headers=headers) as response: + logging.info("list_comments: %s response: %s", url, response) + return response.json() + + +def add_comment(token, issue_number, comment): + """https://docs.github.com/en/rest/reference/issues#create-an-issue-comment""" + url = f'{FIREBASE_URL}/issues/{issue_number}/comments' + headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'} + data = {'body': comment} + with requests.post(url, headers=headers, data=json.dumps(data)) as response: + logging.info("add_comment: %s response: %s", url, response) + + +def update_comment(token, comment_id, comment): + """https://docs.github.com/en/rest/reference/issues#update-an-issue-comment""" + url = f'{FIREBASE_URL}/issues/comments/{comment_id}' + headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'} + data = {'body': comment} + with requests.patch(url, headers=headers, data=json.dumps(data)) as response: + logging.info("update_comment: %s response: %s", url, response) + + +def delete_comment(token, comment_id): + """https://docs.github.com/en/rest/reference/issues#delete-an-issue-comment""" + url = f'{FIREBASE_URL}/issues/comments/{comment_id}' + headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'} + with requests.delete(url, headers=headers) as response: + logging.info("delete_comment: %s response: %s", url, response) + + +def add_label(token, issue_number, label): + """https://docs.github.com/en/rest/reference/issues#add-labels-to-an-issue""" + url = f'{FIREBASE_URL}/issues/{issue_number}/labels' + headers={} + headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'} + data = [label] + with requests.post(url, headers=headers, data=json.dumps(data)) as response: + logging.info("add_label: %s response: %s", url, response) + + +def delete_label(token, issue_number, label): + """https://docs.github.com/en/rest/reference/issues#delete-a-label""" + url = f'{FIREBASE_URL}/issues/{issue_number}/labels/{label}' + headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'} + with requests.delete(url, headers=headers) as response: + logging.info("delete_label: %s response: %s", url, response) + + +def list_artifacts(token, run_id): + """https://docs.github.com/en/rest/reference/actions#list-workflow-run-artifacts""" + url = f'{FIREBASE_URL}/actions/runs/{run_id}/artifacts' + headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'} + with requests.get(url, headers=headers) as response: + logging.info("list_artifacts: %s response: %s", url, response) + return response.json()["artifacts"] + + +def download_artifact(token, artifact_id, output_path): + """https://docs.github.com/en/rest/reference/actions#download-an-artifact""" + url = f'{FIREBASE_URL}/actions/artifacts/{artifact_id}/zip' + headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'} + with requests.get(url, headers=headers, stream=True) as response: + logging.info("download_artifact: %s response: %s", url, response) + with open(output_path, 'wb') as file: + shutil.copyfileobj(response.raw, file) diff --git a/scripts/gha/it_workflow.py b/scripts/gha/it_workflow.py new file mode 100644 index 0000000000..648dd6a1ad --- /dev/null +++ b/scripts/gha/it_workflow.py @@ -0,0 +1,264 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A utility for integration test workflow. + +This script helps to update PR/Issue comments and labels during testing process. + +For PR comment, this script will update (create if not exist) the "Test Result" in comment. +stage value: [start, progress, end] +USAGE: + python scripts/gha/it_workflow.py --stage \ + --token ${{github.token}} \ + --issue_number ${{needs.check_trigger.outputs.pr_number}}\ + --actor ${{github.actor}} \ + --commit ${{needs.prepare_matrix.outputs.github_ref}} \ + --run_id ${{github.run_id}} \ + [--new_token ${{steps.generate-token.outputs.token}}] + +For Daily Report, this script will update (create if not exist) the "Test Result" in Issue +with title "Nightly Integration Testing Report" and label "nightly-testing". +stage value: [report] +USAGE: + python scripts/gha/it_workflow.py --stage report \ + --token ${{github.token}} \ + --actor ${{github.actor}} \ + --commit ${{needs.prepare_matrix.outputs.github_ref}} \ + --run_id ${{github.run_id}} + +""" + +import datetime +import pytz +import shutil + +from absl import app +from absl import flags +from absl import logging + +import github +import summarize_test_results as summarize + +_REPORT_LABEL = "nightly-testing" +_REPORT_TITLE = "Nightly Integration Testing Report" + +_LABEL_PROGRESS = "tests: in-progress" +_LABEL_FAILED = "tests: failed" +_LABEL_SUCCEED = "tests: succeeded" + +_COMMENT_TITLE_PROGESS = "### ⏳  Integration test in progress...\n" +_COMMENT_TITLE_PROGESS_FAIL = "### ❌  Integration test FAILED (but still ⏳  in progress)\n" +_COMMENT_TITLE_FAIL = "### ❌  Integration test FAILED\n" +_COMMENT_TITLE_SUCCEED = "### ✅  Integration test succeeded!\n" + +_COMMENT_IDENTIFIER = "integration-test-status-comment" +_COMMENT_SUFFIX = f'\n\n\n' + +_LOG_ARTIFACT_NAME = "log-artifact" +_LOG_OUTPUT_DIR = "test_results" + +_BUILD_STAGES_START = "start" +_BUILD_STAGES_PROGRESS = "progress" +_BUILD_STAGES_END = "end" +_BUILD_STAGES_REPORT = "report" +_BUILD_STAGES = [_BUILD_STAGES_START, _BUILD_STAGES_PROGRESS, _BUILD_STAGES_END, _BUILD_STAGES_REPORT] + +FLAGS = flags.FLAGS + +flags.DEFINE_string( + "stage", None, + "Different stage while running the workflow. Valid values in _BUILD_STAGES.") + +flags.DEFINE_string( + "token", None, + "github.token: A token to authenticate on your repository.") + +flags.DEFINE_string( + "issue_number", None, + "Github's issue # or pull request #.") + +flags.DEFINE_string( + "actor", None, + "github.actor: The login of the user that initiated the workflow run.") + +flags.DEFINE_string( + "commit", None, "GitHub commit hash") + +flags.DEFINE_string( + "run_id", None, + "github.run_id: A unique number for each workflow run within a repository.") + +flags.DEFINE_string( + "new_token", None, + "Only used with --stage end" + "Use a different token to remove the \"in-progress\" label," + "to allow the removal to trigger the \"Check Labels\" workflow.") + +def test_start(token, issue_number, actor, commit, run_id): + """In PR, when start testing, add comment and label \"tests: in-progress\"""" + github.add_label(token, issue_number, _LABEL_PROGRESS) + for label in [_LABEL_FAILED, _LABEL_SUCCEED]: + github.delete_label(token, issue_number, label) + + comment = (_COMMENT_TITLE_PROGESS + + _get_description(actor, commit, run_id) + + _COMMENT_SUFFIX) + _update_comment(token, issue_number, comment) + + +def test_progress(token, issue_number, actor, commit, run_id): + """In PR, when some test failed, update failure info and + add label \"tests: failed\"""" + log_summary = _get_summary_talbe(token, run_id) + if log_summary == 0: + return + else: + github.add_label(token, issue_number, _LABEL_FAILED) + comment = (_COMMENT_TITLE_PROGESS_FAIL + + _get_description(actor, commit, run_id) + + log_summary + + _COMMENT_SUFFIX) + _update_comment(token, issue_number, comment) + + +def test_end(token, issue_number, actor, commit, run_id, new_token): + """In PR, when some test end, update Test Result Report and + update label: add \"tests: failed\" if test failed, add label + \"tests: succeeded\" if test succeed""" + log_summary = _get_summary_talbe(token, run_id) + if log_summary == 0: + github.add_label(token, issue_number, _LABEL_SUCCEED) + comment = (_COMMENT_TITLE_SUCCEED + + _get_description(actor, commit, run_id) + + _COMMENT_SUFFIX) + _update_comment(token, issue_number, comment) + else: + github.add_label(token, issue_number, _LABEL_FAILED) + comment = (_COMMENT_TITLE_FAIL + + _get_description(actor, commit, run_id) + + log_summary + + _COMMENT_SUFFIX) + _update_comment(token, issue_number, comment) + + github.delete_label(new_token, issue_number, _LABEL_PROGRESS) + + +def test_report(token, actor, commit, run_id): + """Update (create if not exist) a Daily Report in Issue. + If test failed, add label \"tests: failed\" and open the Issue, + If test succeed, add label \"tests: succeeded\" and close the Issue. + The Issue with title _REPORT_TITLE and label _REPORT_LABEL: + https://github.com/firebase/firebase-cpp-sdk/issues?q=is%3Aissue+is%3Aclosed+label%3Anightly-testing + """ + issue_number = _get_issue_number(token, _REPORT_TITLE, _REPORT_LABEL) + log_summary = _get_summary_talbe(token, run_id) + if log_summary == 0: +# github.delete_label(token, issue_number, _LABEL_FAILED) +# github.add_label(token, issue_number, _LABEL_SUCCEED) + github.close_issue(token, issue_number) + comment = (_COMMENT_TITLE_SUCCEED + + _get_description(actor, commit, run_id) + + _COMMENT_SUFFIX) + github.update_issue_comment(token, issue_number, comment) + else: +# github.delete_label(token, issue_number, _LABEL_SUCCEED) +# github.add_label(token, issue_number, _LABEL_FAILED) + github.open_issue(token, issue_number) + comment = (_COMMENT_TITLE_FAIL + + _get_description(actor, commit, run_id) + + log_summary + + _COMMENT_SUFFIX) + github.update_issue_comment(token, issue_number, comment) + + +def _get_issue_number(token, title, label): + issues = github.search_issues_by_label(label) + for issue in issues: + if issue["title"] == title: + return issue["number"] + + return github.create_issue(token, title, label)["number"] + + +def _update_comment(token, issue_number, comment): + comment_id = _get_comment_id(issue_number, _COMMENT_SUFFIX) + if not comment_id: + github.add_comment(token, issue_number, comment) + else: + github.update_comment(token, comment_id, comment) + + +def _get_comment_id(issue_number, comment_identifier): + comments = github.list_comments(issue_number) + for comment in comments: + if comment_identifier in comment['body']: + return comment['id'] + return None + + +def _get_description(actor, commit, run_id): + """Test Result Report Title and description""" + return ("Requested by @%s on commit %s\n" % (actor, commit) + + "Last updated: %s \n" % _get_datetime() + + "**[View integration test log & download artifacts](https://github.com/firebase/firebase-cpp-sdk/actions/runs/%s)**\n" % run_id) + + +def _get_datetime(): + """Date time when Test Result Report updated""" + pst_now = datetime.datetime.utcnow().astimezone(pytz.timezone("America/Los_Angeles")) + return pst_now.strftime("%a %b %e %H:%M %Z %G") + + +def _get_summary_talbe(token, run_id): + """Test Result Report Body, which is failed test table with markdown format""" + # artifact_id only exist after workflow finishs running + # Thus, "down artifact" logic is in the workflow + # artifact_id = _get_artifact_id(token, run_id, _LOG_ARTIFACT_NAME) + # artifact_path = _LOG_ARTIFACT_NAME + ".zip" + # github.download_artifact(token, artifact_id, artifact_path) + # shutil.unpack_archive(artifact_path, _LOG_OUTPUT_DIR) + summary_talbe = summarize.summarize_logs(dir=_LOG_OUTPUT_DIR, markdown=True) + return summary_talbe + + +def _get_artifact_id(token, run_id, name): + artifacts = github.list_artifacts(token, run_id) + for artifact in artifacts: + if artifact["name"] == name: + return artifact["id"] + + +def main(argv): + if len(argv) > 1: + raise app.UsageError("Too many command-line arguments.") + + if FLAGS.stage == _BUILD_STAGES_START: + test_start(FLAGS.token, FLAGS.issue_number, FLAGS.actor, FLAGS.commit, FLAGS.run_id) + elif FLAGS.stage == _BUILD_STAGES_PROGRESS: + test_progress(FLAGS.token, FLAGS.issue_number, FLAGS.actor, FLAGS.commit, FLAGS.run_id) + elif FLAGS.stage == _BUILD_STAGES_END: + test_end(FLAGS.token, FLAGS.issue_number, FLAGS.actor, FLAGS.commit, FLAGS.run_id, FLAGS.new_token) + elif FLAGS.stage == _BUILD_STAGES_REPORT: + test_report(FLAGS.token, FLAGS.actor, FLAGS.commit, FLAGS.run_id) + else: + print("Invalid stage value. Valid value: " + ",".join(_BUILD_STAGES)) + + +if __name__ == "__main__": + flags.mark_flag_as_required("stage") + flags.mark_flag_as_required("token") + flags.mark_flag_as_required("actor") + flags.mark_flag_as_required("commit") + flags.mark_flag_as_required("run_id") + app.run(main) diff --git a/scripts/gha/requirements.txt b/scripts/gha/requirements.txt index d265d0774c..5ec721cd3a 100644 --- a/scripts/gha/requirements.txt +++ b/scripts/gha/requirements.txt @@ -1,2 +1,4 @@ absl-py attrs +pytz +requests diff --git a/scripts/gha/summarize_test_results.py b/scripts/gha/summarize_test_results.py index d04165362f..c16f103ce4 100644 --- a/scripts/gha/summarize_test_results.py +++ b/scripts/gha/summarize_test_results.py @@ -69,9 +69,7 @@ "github_log", False, "Display a GitHub log list.") -flags.DEFINE_integer( - "list_max", 3, - "In Markdown mode, collapse lists larger than this size. 0 to disable.") +LIST_MAX = 3 CAPITALIZATIONS = { "macos": "MacOS", @@ -136,7 +134,7 @@ def format_result(config_tests, space_char = " ", list_separator=DEFAULT_LIST_SE else: config_set.add(config+list_separator) - if FLAGS.markdown and FLAGS.list_max > 0 and len(config_set) > FLAGS.list_max: + if len(config_set) > LIST_MAX: return "
(%s items)%s
" % ( len(config_set), "".join(sorted(config_set))) else: @@ -179,15 +177,18 @@ def print_markdown_table(log_results): def main(argv): if len(argv) > 1: raise app.UsageError("Too many command-line arguments.") + summarize_logs(FLAGS.dir, FLAGS.markdown, FLAGS.github_log) - build_log_files = glob.glob(os.path.join(FLAGS.dir, BUILD_FILE_PATTERN)) - test_log_files = glob.glob(os.path.join(FLAGS.dir, TEST_FILE_PATTERN)) + +def summarize_logs(dir, markdown=False, github_log=False): + build_log_files = glob.glob(os.path.join(dir, BUILD_FILE_PATTERN)) + test_log_files = glob.glob(os.path.join(dir, TEST_FILE_PATTERN)) # Replace the "*" in the file glob with a regex capture group, # so we can report the test name. build_log_name_re = re.escape( - os.path.join(FLAGS.dir,BUILD_FILE_PATTERN)).replace("\\*", "(.*)") + os.path.join(dir,BUILD_FILE_PATTERN)).replace("\\*", "(.*)") test_log_name_re = re.escape( - os.path.join(FLAGS.dir,TEST_FILE_PATTERN)).replace("\\*", "(.*)") + os.path.join(dir,TEST_FILE_PATTERN)).replace("\\*", "(.*)") any_failures = False log_data = {} @@ -224,15 +225,17 @@ def main(argv): return(0) log_lines = [] - if FLAGS.markdown: + if markdown: log_lines = print_markdown_table(log_results) # If outputting Markdown, don't bother justifying the table. - elif FLAGS.github_log: + elif github_log: log_lines = print_github_log(log_results) else: log_lines = print_log(log_results) - print("\n".join(log_lines)) + log_summary = "\n".join(log_lines) + print(log_summary) + return log_summary def get_configs_from_file_name(file_name, file_name_re):