diff --git a/.github/workflows/triage-unallowed-contributions.yml b/.github/workflows/triage-unallowed-contributions.yml
index 662d1f19a1d6..9cbee56a77a9 100644
--- a/.github/workflows/triage-unallowed-contributions.yml
+++ b/.github/workflows/triage-unallowed-contributions.yml
@@ -104,7 +104,8 @@ jobs:
               body: reviewMessage,
               event: 'REQUEST_CHANGES'
             })
-            exit 1 # prevents further steps from running and fails workflow
+
+            core.setFailed("It looks like you've modified some files we don't accept contributions for. Please see the review with requested changes for details.")
       # When the most recent review was CHANGES_REQUESTED and the existing
       # PR no longer contains unallowed changes, dismiss the previous review
       - name: Dismiss pull request review