diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index ceda84c21acc..976a81c9035c 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -231,17 +231,9 @@ jobs:
         with:
           path: ${{ env.CACHED_BUILD_PATHS }}
           key: ${{ env.BUILD_CACHE_KEY }}
-      - name: Check changed files
-        id: changed-files-specific
-        uses: tj-actions/changed-files@v6.2
-        with:
-          files: .*packages\/ember($|/.*)
-      # Only run ember tests if the files above have changed
       - name: Run Ember tests
-        if: steps.changed-files-specific.outputs.any_changed == 'true' || github.event_name == 'push'
         run: yarn test --scope=@sentry/ember
       - name: Compute test coverage
-        if: steps.changed-files-specific.outputs.any_changed == 'true' || github.event_name == 'push'
         uses: codecov/codecov-action@v1
 
   job_artifacts:
diff --git a/packages/ember/config/ember-try.js b/packages/ember/config/ember-try.js
index e54fd4886fbb..94f64eb39117 100644
--- a/packages/ember/config/ember-try.js
+++ b/packages/ember/config/ember-try.js
@@ -3,26 +3,47 @@
 const getChannelURL = require('ember-source-channel-url');
 const { embroiderSafe } = require('@embroider/test-setup');
 
+/**
+ * Pick which versions of ember against which to test based on whether the tests are running locally, as part of a PR,
+ * or when merging to `master` or creating a release.
+ *
+ * @returns The versions which should be tested, along with their respective config
+ */
 module.exports = async function() {
-  return {
-    useYarn: true,
-    scenarios: [
-      {
-        name: 'ember-lts-3.20',
-        npm: {
-          devDependencies: {
-            'ember-source': '~3.20.0',
-          },
+  // whenever and wherever we test, we want to at least test against the latest version of ember
+  let scenarios = [
+    {
+      name: 'ember-release',
+      npm: {
+        devDependencies: {
+          'ember-source': await getChannelURL('release'),
         },
       },
+    },
+  ];
+
+  // in CI we add a few more tests - LTS and embroider (which is an ember compiler)
+  if (process.env.GITHUB_ACTIONS) {
+    scenarios = scenarios.concat([
       {
-        name: 'ember-release',
+        name: 'ember-lts-3.20',
         npm: {
           devDependencies: {
-            'ember-source': await getChannelURL('release'),
+            'ember-source': '~3.24.0',
           },
         },
       },
+      embroiderSafe(),
+    ]);
+  }
+
+  // finally, just to be extra thorough when merging to master and releasing, we add the beta channel and ember
+  // "classic" (a legacy version which was last current in late 2019)
+  if (
+    process.env.GITHUB_EVENT_NAME === 'push' &&
+    (process.env.GITHUB_HEAD_REF === 'master' || process.env.GITHUB_HEAD_REF.startsWith('release'))
+  ) {
+    scenarios = scenarios.concat([
       {
         name: 'ember-beta',
         npm: {
@@ -32,7 +53,6 @@ module.exports = async function() {
         },
         allowedToFail: true,
       },
-      embroiderSafe(),
       {
         name: 'ember-classic',
         env: {
@@ -48,6 +68,11 @@ module.exports = async function() {
           },
         },
       },
-    ],
+    ]);
+  }
+
+  return {
+    useYarn: true,
+    scenarios,
   };
 };
diff --git a/packages/ember/package.json b/packages/ember/package.json
index 3915b94ade45..02968b4161b7 100644
--- a/packages/ember/package.json
+++ b/packages/ember/package.json
@@ -26,9 +26,8 @@
     "prepublishOnly": "ember ts:precompile",
     "postpublish": "ember ts:clean",
     "start": "ember serve",
-    "test": "node ./scripts/run_tests.js",
-    "test:ember": "ember test",
-    "test:ember-compatibility": "ember try:each"
+    "test": "ember try:each",
+    "test:all": "node ./scripts/run-CI-tests.js"
   },
   "dependencies": {
     "@embroider/macros": "~0.47.2",
diff --git a/packages/ember/scripts/run-CI-tests.js b/packages/ember/scripts/run-CI-tests.js
new file mode 100644
index 000000000000..a521ecafd328
--- /dev/null
+++ b/packages/ember/scripts/run-CI-tests.js
@@ -0,0 +1,17 @@
+/*eslint-env node*/
+const { spawnSync } = require('child_process');
+
+console.log('Mimicking the CI environment in order to run tests against multiple versions of Ember');
+
+const result = spawnSync('yarn test', {
+  shell: true,
+  stdio: 'inherit',
+  env: {
+    ...process.env,
+    GITHUB_ACTIONS: true,
+    GITHUB_EVENT_NAME: 'push',
+    GITHUB_HEAD_REF: 'master',
+  },
+});
+
+process.exit(result.status);
diff --git a/packages/ember/scripts/run_tests.js b/packages/ember/scripts/run_tests.js
deleted file mode 100644
index fb594689893a..000000000000
--- a/packages/ember/scripts/run_tests.js
+++ /dev/null
@@ -1,15 +0,0 @@
-// running compatibilty tests takes ~15 min on a 2019 2.6 GHz 6-Core Intel i7 16" MacBook Pro w 32 GB of RAM, vs ~25 sec
-// for the regular tests
-
-/*eslint-env node*/
-const { spawnSync } = require('child_process');
-
-if (process.env.TRAVIS || process.env.GITHUB_ACTIONS) {
-  console.log('In CI - running tests against multiple versions of Ember');
-  const result = spawnSync('yarn npm-run-all lint:* test:*', { shell: true, stdio: 'inherit' });
-  process.exit(result.status);
-} else {
-  console.log('Tests running locally - will only run tests against default version of Ember');
-  const result = spawnSync('yarn npm-run-all lint:* test:ember', { shell: true, stdio: 'inherit' });
-  process.exit(result.status);
-}