From 30e978bad8851867d46db7358e4469102d477e9e Mon Sep 17 00:00:00 2001
From: Luca Forstner <luca.forstner@sentry.io>
Date: Tue, 27 Sep 2022 15:40:31 +0000
Subject: [PATCH 01/14] test(e2e): Add E2E Test recipe framework

---
 packages/e2e-tests/run.ts | 91 +++++++++++++++++++++++----------------
 1 file changed, 54 insertions(+), 37 deletions(-)

diff --git a/packages/e2e-tests/run.ts b/packages/e2e-tests/run.ts
index 4a752e15a294..b578e44223c6 100644
--- a/packages/e2e-tests/run.ts
+++ b/packages/e2e-tests/run.ts
@@ -11,42 +11,59 @@ const PUBLISH_PACKAGES_DOCKER_IMAGE_NAME = 'publish-packages';
 
 const publishScriptNodeVersion = process.env.E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION;
 
-try {
-  // Stop test registry container (Verdaccio) if it was already running
-  childProcess.execSync(`docker stop ${TEST_REGISTRY_CONTAINER_NAME}`, { stdio: 'ignore' });
-  console.log('Stopped previously running test registry');
-} catch (e) {
-  // Don't throw if container wasn't running
+// https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines
+function groupCIOutput(groupTitle: string, fn: () => void): void {
+  if (process.env.CI) {
+    console.log(`::group::{${groupTitle}}`);
+    fn();
+    console.log('::endgroup::');
+  } else {
+    fn();
+  }
 }
 
-// Start test registry (Verdaccio)
-childProcess.execSync(
-  `docker run --detach --rm --name ${TEST_REGISTRY_CONTAINER_NAME} -p 4873:4873 -v ${__dirname}/verdaccio-config:/verdaccio/conf verdaccio/verdaccio:${VERDACCIO_VERSION}`,
-  { encoding: 'utf8', stdio: 'inherit' },
-);
-
-// Build container image that is uploading our packages to fake registry with specific Node.js/npm version
-childProcess.execSync(
-  `docker build --tag ${PUBLISH_PACKAGES_DOCKER_IMAGE_NAME} --file ./Dockerfile.publish-packages ${
-    publishScriptNodeVersion ? `--build-arg NODE_VERSION=${publishScriptNodeVersion}` : ''
-  } .`,
-  {
-    encoding: 'utf8',
-    stdio: 'inherit',
-  },
-);
-
-// Run container that uploads our packages to fake registry
-childProcess.execSync(
-  `docker run --rm -v ${repositoryRoot}:/sentry-javascript --network host ${PUBLISH_PACKAGES_DOCKER_IMAGE_NAME}`,
-  {
-    encoding: 'utf8',
-    stdio: 'inherit',
-  },
-);
-
-// TODO: Run e2e tests here
-
-// Stop test registry
-childProcess.execSync(`docker stop ${TEST_REGISTRY_CONTAINER_NAME}`, { encoding: 'utf8', stdio: 'ignore' });
-console.log('Successfully stopped test registry container'); // Output from command above is not good so we `ignore` it and emit our own
+groupCIOutput('Test Registry Setup', () => {
+  try {
+    // Stop test registry container (Verdaccio) if it was already running
+    childProcess.execSync(`docker stop ${TEST_REGISTRY_CONTAINER_NAME}`, { stdio: 'ignore' });
+    console.log('Stopped previously running test registry');
+  } catch (e) {
+    // Don't throw if container wasn't running
+  }
+
+  // Start test registry (Verdaccio)
+  childProcess.execSync(
+    `docker run --detach --rm --name ${TEST_REGISTRY_CONTAINER_NAME} -p 4873:4873 -v ${__dirname}/verdaccio-config:/verdaccio/conf verdaccio/verdaccio:${VERDACCIO_VERSION}`,
+    { encoding: 'utf8', stdio: 'inherit' },
+  );
+
+  // Build container image that is uploading our packages to fake registry with specific Node.js/npm version
+  childProcess.execSync(
+    `docker build --tag ${PUBLISH_PACKAGES_DOCKER_IMAGE_NAME} --file ./Dockerfile.publish-packages ${
+      publishScriptNodeVersion ? `--build-arg NODE_VERSION=${publishScriptNodeVersion}` : ''
+    } .`,
+    {
+      encoding: 'utf8',
+      stdio: 'inherit',
+    },
+  );
+
+  // Run container that uploads our packages to fake registry
+  childProcess.execSync(
+    `docker run --rm -v ${repositoryRoot}:/sentry-javascript --network host ${PUBLISH_PACKAGES_DOCKER_IMAGE_NAME}`,
+    {
+      encoding: 'utf8',
+      stdio: 'inherit',
+    },
+  );
+});
+
+groupCIOutput('Run E2E Test Suites', () => {
+  // TODO: Run e2e tests here
+});
+
+groupCIOutput('Cleanup', () => {
+  // Stop test registry
+  childProcess.execSync(`docker stop ${TEST_REGISTRY_CONTAINER_NAME}`, { encoding: 'utf8', stdio: 'ignore' });
+  console.log('Successfully stopped test registry container'); // Output from command above is not good so we `ignore` it and emit our own
+});

From 908b2b9bd9359beb619eb43d89b19e2d03e94f46 Mon Sep 17 00:00:00 2001
From: Luca Forstner <luca.forstner@sentry.io>
Date: Wed, 28 Sep 2022 11:34:57 +0000
Subject: [PATCH 02/14] Add logic

---
 packages/e2e-tests/.eslintrc.js               |   2 +-
 packages/e2e-tests/run.ts                     | 160 ++++++++++++++++--
 .../temporary-app-1/.gitignore                |   1 +
 .../test-applications/temporary-app-1/.npmrc  |   3 +
 .../test-applications/temporary-app-1/bad.js  |   1 +
 .../test-applications/temporary-app-1/good.js |   1 +
 .../temporary-app-1/package.json              |  13 ++
 .../temporary-app-1/test-recipe.json          |  21 +++
 .../temporary-app-1/timeout.js                |   3 +
 .../temporary-app-2/.gitignore                |   1 +
 .../test-applications/temporary-app-2/.npmrc  |   3 +
 .../test-applications/temporary-app-2/bad.js  |   1 +
 .../test-applications/temporary-app-2/good.js |   1 +
 .../temporary-app-2/package.json              |  13 ++
 .../temporary-app-2/test-recipe.json          |  21 +++
 .../temporary-app-2/timeout.js                |   3 +
 packages/e2e-tests/test-recipe-schema.json    |  38 +++++
 17 files changed, 274 insertions(+), 12 deletions(-)
 create mode 100644 packages/e2e-tests/test-applications/temporary-app-1/.gitignore
 create mode 100644 packages/e2e-tests/test-applications/temporary-app-1/.npmrc
 create mode 100644 packages/e2e-tests/test-applications/temporary-app-1/bad.js
 create mode 100644 packages/e2e-tests/test-applications/temporary-app-1/good.js
 create mode 100644 packages/e2e-tests/test-applications/temporary-app-1/package.json
 create mode 100644 packages/e2e-tests/test-applications/temporary-app-1/test-recipe.json
 create mode 100644 packages/e2e-tests/test-applications/temporary-app-1/timeout.js
 create mode 100644 packages/e2e-tests/test-applications/temporary-app-2/.gitignore
 create mode 100644 packages/e2e-tests/test-applications/temporary-app-2/.npmrc
 create mode 100644 packages/e2e-tests/test-applications/temporary-app-2/bad.js
 create mode 100644 packages/e2e-tests/test-applications/temporary-app-2/good.js
 create mode 100644 packages/e2e-tests/test-applications/temporary-app-2/package.json
 create mode 100644 packages/e2e-tests/test-applications/temporary-app-2/test-recipe.json
 create mode 100644 packages/e2e-tests/test-applications/temporary-app-2/timeout.js
 create mode 100644 packages/e2e-tests/test-recipe-schema.json

diff --git a/packages/e2e-tests/.eslintrc.js b/packages/e2e-tests/.eslintrc.js
index 5b0457483479..8c585e48f252 100644
--- a/packages/e2e-tests/.eslintrc.js
+++ b/packages/e2e-tests/.eslintrc.js
@@ -3,7 +3,7 @@ module.exports = {
     node: true,
   },
   extends: ['../../.eslintrc.js'],
-  ignorePatterns: [],
+  ignorePatterns: ['test-applications/**'],
   parserOptions: {
     sourceType: 'module',
   },
diff --git a/packages/e2e-tests/run.ts b/packages/e2e-tests/run.ts
index b578e44223c6..2acaee570f23 100644
--- a/packages/e2e-tests/run.ts
+++ b/packages/e2e-tests/run.ts
@@ -1,5 +1,7 @@
 /* eslint-disable no-console */
 import * as childProcess from 'child_process';
+import * as fs from 'fs';
+import * as glob from 'glob';
 import * as path from 'path';
 
 const repositoryRoot = path.resolve(__dirname, '../..');
@@ -11,10 +13,12 @@ const PUBLISH_PACKAGES_DOCKER_IMAGE_NAME = 'publish-packages';
 
 const publishScriptNodeVersion = process.env.E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION;
 
+const DEFAULT_TEST_TIMEOUT_SECONDS = 60;
+
 // https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines
 function groupCIOutput(groupTitle: string, fn: () => void): void {
   if (process.env.CI) {
-    console.log(`::group::{${groupTitle}}`);
+    console.log(`::group::${groupTitle}`);
     fn();
     console.log('::endgroup::');
   } else {
@@ -22,26 +26,54 @@ function groupCIOutput(groupTitle: string, fn: () => void): void {
   }
 }
 
+// https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message
+function printCIErrorMessage(message: string): void {
+  if (process.env.CI) {
+    console.log(`::error::${message}`);
+  } else {
+    console.log(message);
+  }
+}
+
 groupCIOutput('Test Registry Setup', () => {
   try {
     // Stop test registry container (Verdaccio) if it was already running
-    childProcess.execSync(`docker stop ${TEST_REGISTRY_CONTAINER_NAME}`, { stdio: 'ignore' });
+    childProcess.spawnSync('docker', ['stop', TEST_REGISTRY_CONTAINER_NAME], { stdio: 'ignore' });
     console.log('Stopped previously running test registry');
   } catch (e) {
     // Don't throw if container wasn't running
   }
 
   // Start test registry (Verdaccio)
-  childProcess.execSync(
-    `docker run --detach --rm --name ${TEST_REGISTRY_CONTAINER_NAME} -p 4873:4873 -v ${__dirname}/verdaccio-config:/verdaccio/conf verdaccio/verdaccio:${VERDACCIO_VERSION}`,
+  childProcess.spawnSync(
+    'docker',
+    [
+      'run',
+      '--detach',
+      '--rm',
+      '--name',
+      TEST_REGISTRY_CONTAINER_NAME,
+      '-p',
+      '4873:4873',
+      '-v',
+      `${__dirname}/verdaccio-config:/verdaccio/conf`,
+      `verdaccio/verdaccio:${VERDACCIO_VERSION}`,
+    ],
     { encoding: 'utf8', stdio: 'inherit' },
   );
 
   // Build container image that is uploading our packages to fake registry with specific Node.js/npm version
-  childProcess.execSync(
-    `docker build --tag ${PUBLISH_PACKAGES_DOCKER_IMAGE_NAME} --file ./Dockerfile.publish-packages ${
-      publishScriptNodeVersion ? `--build-arg NODE_VERSION=${publishScriptNodeVersion}` : ''
-    } .`,
+  childProcess.spawnSync(
+    'docker',
+    [
+      'build',
+      '--tag',
+      PUBLISH_PACKAGES_DOCKER_IMAGE_NAME,
+      '--file',
+      './Dockerfile.publish-packages',
+      publishScriptNodeVersion ? `--build-arg NODE_VERSION=${publishScriptNodeVersion}` : undefined,
+      '.',
+    ].filter((arg): arg is string => arg !== undefined),
     {
       encoding: 'utf8',
       stdio: 'inherit',
@@ -49,8 +81,17 @@ groupCIOutput('Test Registry Setup', () => {
   );
 
   // Run container that uploads our packages to fake registry
-  childProcess.execSync(
-    `docker run --rm -v ${repositoryRoot}:/sentry-javascript --network host ${PUBLISH_PACKAGES_DOCKER_IMAGE_NAME}`,
+  childProcess.spawnSync(
+    'docker',
+    [
+      'run',
+      '--rm',
+      '-v',
+      `${repositoryRoot}:/sentry-javascript`,
+      '--network',
+      'host',
+      PUBLISH_PACKAGES_DOCKER_IMAGE_NAME,
+    ],
     {
       encoding: 'utf8',
       stdio: 'inherit',
@@ -60,10 +101,107 @@ groupCIOutput('Test Registry Setup', () => {
 
 groupCIOutput('Run E2E Test Suites', () => {
   // TODO: Run e2e tests here
+  const recipePaths = glob.sync(`${__dirname}/test-applications/*/test-recipe.json`, { absolute: true });
+
+  const recipeResults = recipePaths.map(recipePath => {
+    type Recipe = {
+      testApplicationName: string;
+      buildCommand?: string;
+      tests: {
+        testName: string;
+        testCommand: string;
+        timeoutSeconds?: number;
+      }[];
+    };
+
+    const recipe: Recipe = JSON.parse(fs.readFileSync(recipePath, 'utf-8'));
+
+    if (recipe.buildCommand) {
+      console.log(`Running E2E test build command for test application "${recipe.testApplicationName}"`);
+      const [buildCommand, ...buildCommandArgs] = recipe.buildCommand.split(' ');
+      childProcess.spawnSync(buildCommand, buildCommandArgs, {
+        cwd: path.dirname(recipePath),
+        encoding: 'utf8',
+        stdio: 'inherit',
+      });
+    }
+
+    type TestResult = {
+      testName: string;
+      result: 'PASS' | 'FAIL' | 'TIMEOUT';
+    };
+
+    const testResults: TestResult[] = recipe.tests.map(test => {
+      console.log(
+        `Running E2E test command for test application "${recipe.testApplicationName}", test "${test.testName}"`,
+      );
+
+      const [testCommand, ...testCommandArgs] = test.testCommand.split(' ');
+      const testProcessResult = childProcess.spawnSync(testCommand, testCommandArgs, {
+        cwd: path.dirname(recipePath),
+        timeout: (test.timeoutSeconds ?? DEFAULT_TEST_TIMEOUT_SECONDS) * 1000,
+        encoding: 'utf8',
+        stdio: 'pipe',
+      });
+
+      console.log(testProcessResult.stdout.replace(/^/gm, '[TEST OUTPUT] '));
+      console.log(testProcessResult.stderr.replace(/^/gm, '[TEST OUTPUT] '));
+
+      const error: undefined | (Error & { code?: string }) = testProcessResult.error;
+
+      if (error?.code === 'ETIMEDOUT') {
+        printCIErrorMessage(
+          `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname(
+            recipePath,
+          )}) timed out.`,
+        );
+        return {
+          testName: test.testName,
+          result: 'TIMEOUT',
+        };
+      } else if (testProcessResult.status !== 0) {
+        printCIErrorMessage(
+          `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname(
+            recipePath,
+          )}) failed.`,
+        );
+        return {
+          testName: test.testName,
+          result: 'FAIL',
+        };
+      } else {
+        console.log(
+          `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname(
+            recipePath,
+          )}) succeeded.`,
+        );
+        return {
+          testName: test.testName,
+          result: 'PASS',
+        };
+      }
+    });
+
+    return {
+      testApplicationName: recipe.testApplicationName,
+      testApplicationPath: recipePath,
+      testResults,
+    };
+  });
+
+  console.log('--------------------------------------');
+  console.log('Test Result Summary:');
+
+  recipeResults.forEach(recipeResult => {
+    console.log(`● ${recipeResult.testApplicationName} (${path.dirname(recipeResult.testApplicationPath)})`);
+    recipeResult.testResults.forEach(testResult => {
+      console.log(`  ● ${testResult.result.padEnd(7, ' ')} ${testResult.testName}`);
+    });
+  });
 });
 
 groupCIOutput('Cleanup', () => {
   // Stop test registry
-  childProcess.execSync(`docker stop ${TEST_REGISTRY_CONTAINER_NAME}`, { encoding: 'utf8', stdio: 'ignore' });
+  childProcess.spawnSync(`docker stop ${TEST_REGISTRY_CONTAINER_NAME}`, { encoding: 'utf8', stdio: 'ignore' });
   console.log('Successfully stopped test registry container'); // Output from command above is not good so we `ignore` it and emit our own
 });
diff --git a/packages/e2e-tests/test-applications/temporary-app-1/.gitignore b/packages/e2e-tests/test-applications/temporary-app-1/.gitignore
new file mode 100644
index 000000000000..8ee01d321b72
--- /dev/null
+++ b/packages/e2e-tests/test-applications/temporary-app-1/.gitignore
@@ -0,0 +1 @@
+yarn.lock
diff --git a/packages/e2e-tests/test-applications/temporary-app-1/.npmrc b/packages/e2e-tests/test-applications/temporary-app-1/.npmrc
new file mode 100644
index 000000000000..c35d987cca9f
--- /dev/null
+++ b/packages/e2e-tests/test-applications/temporary-app-1/.npmrc
@@ -0,0 +1,3 @@
+@sentry:registry=http://localhost:4873
+@sentry-internal:registry=http://localhost:4873
+//localhost:4873/:_authToken=some-token
diff --git a/packages/e2e-tests/test-applications/temporary-app-1/bad.js b/packages/e2e-tests/test-applications/temporary-app-1/bad.js
new file mode 100644
index 000000000000..bf99992592d6
--- /dev/null
+++ b/packages/e2e-tests/test-applications/temporary-app-1/bad.js
@@ -0,0 +1 @@
+throw new Error('Sad :(');
diff --git a/packages/e2e-tests/test-applications/temporary-app-1/good.js b/packages/e2e-tests/test-applications/temporary-app-1/good.js
new file mode 100644
index 000000000000..8da91c072fb8
--- /dev/null
+++ b/packages/e2e-tests/test-applications/temporary-app-1/good.js
@@ -0,0 +1 @@
+console.log('Happy :)');
diff --git a/packages/e2e-tests/test-applications/temporary-app-1/package.json b/packages/e2e-tests/test-applications/temporary-app-1/package.json
new file mode 100644
index 000000000000..449dbd698c88
--- /dev/null
+++ b/packages/e2e-tests/test-applications/temporary-app-1/package.json
@@ -0,0 +1,13 @@
+{
+  "name": "temporary-app-1",
+  "version": "1.0.0",
+  "private": true,
+  "scripts": {
+    "start:good": "node good.js",
+    "start:bad": "node bad.js",
+    "start:timeout": "node timeout.js"
+  },
+  "dependencies": {
+    "@sentry/node": "*"
+  }
+}
diff --git a/packages/e2e-tests/test-applications/temporary-app-1/test-recipe.json b/packages/e2e-tests/test-applications/temporary-app-1/test-recipe.json
new file mode 100644
index 000000000000..ddc1f2e72d9e
--- /dev/null
+++ b/packages/e2e-tests/test-applications/temporary-app-1/test-recipe.json
@@ -0,0 +1,21 @@
+{
+  "$schema": "../../test-recipe-schema.json",
+  "testApplicationName": "Temporary Application 1",
+  "buildCommand": "yarn install",
+  "tests": [
+    {
+      "testName": "Example Test (Should Succeed)",
+      "testCommand": "yarn start:good",
+      "timeoutSeconds": 30
+    },
+    {
+      "testName": "Example Test (Should Fail)",
+      "testCommand": "yarn start:bad"
+    },
+    {
+      "testName": "Example Test (Should time out)",
+      "testCommand": "yarn start:timeout",
+      "timeoutSeconds": 5
+    }
+  ]
+}
diff --git a/packages/e2e-tests/test-applications/temporary-app-1/timeout.js b/packages/e2e-tests/test-applications/temporary-app-1/timeout.js
new file mode 100644
index 000000000000..044ab5d7191a
--- /dev/null
+++ b/packages/e2e-tests/test-applications/temporary-app-1/timeout.js
@@ -0,0 +1,3 @@
+setTimeout(() => {
+  console.log('Bored :/');
+}, 6000);
diff --git a/packages/e2e-tests/test-applications/temporary-app-2/.gitignore b/packages/e2e-tests/test-applications/temporary-app-2/.gitignore
new file mode 100644
index 000000000000..8ee01d321b72
--- /dev/null
+++ b/packages/e2e-tests/test-applications/temporary-app-2/.gitignore
@@ -0,0 +1 @@
+yarn.lock
diff --git a/packages/e2e-tests/test-applications/temporary-app-2/.npmrc b/packages/e2e-tests/test-applications/temporary-app-2/.npmrc
new file mode 100644
index 000000000000..c35d987cca9f
--- /dev/null
+++ b/packages/e2e-tests/test-applications/temporary-app-2/.npmrc
@@ -0,0 +1,3 @@
+@sentry:registry=http://localhost:4873
+@sentry-internal:registry=http://localhost:4873
+//localhost:4873/:_authToken=some-token
diff --git a/packages/e2e-tests/test-applications/temporary-app-2/bad.js b/packages/e2e-tests/test-applications/temporary-app-2/bad.js
new file mode 100644
index 000000000000..bf99992592d6
--- /dev/null
+++ b/packages/e2e-tests/test-applications/temporary-app-2/bad.js
@@ -0,0 +1 @@
+throw new Error('Sad :(');
diff --git a/packages/e2e-tests/test-applications/temporary-app-2/good.js b/packages/e2e-tests/test-applications/temporary-app-2/good.js
new file mode 100644
index 000000000000..8da91c072fb8
--- /dev/null
+++ b/packages/e2e-tests/test-applications/temporary-app-2/good.js
@@ -0,0 +1 @@
+console.log('Happy :)');
diff --git a/packages/e2e-tests/test-applications/temporary-app-2/package.json b/packages/e2e-tests/test-applications/temporary-app-2/package.json
new file mode 100644
index 000000000000..b9a5cb6f68e7
--- /dev/null
+++ b/packages/e2e-tests/test-applications/temporary-app-2/package.json
@@ -0,0 +1,13 @@
+{
+  "name": "temporary-app-2",
+  "version": "1.0.0",
+  "private": true,
+  "scripts": {
+    "start:good": "node good.js",
+    "start:bad": "node bad.js",
+    "start:timeout": "node timeout.js"
+  },
+  "dependencies": {
+    "@sentry/node": "*"
+  }
+}
diff --git a/packages/e2e-tests/test-applications/temporary-app-2/test-recipe.json b/packages/e2e-tests/test-applications/temporary-app-2/test-recipe.json
new file mode 100644
index 000000000000..9d93f49cc667
--- /dev/null
+++ b/packages/e2e-tests/test-applications/temporary-app-2/test-recipe.json
@@ -0,0 +1,21 @@
+{
+  "$schema": "../../test-recipe-schema.json",
+  "testApplicationName": "Temporary Application 2",
+  "buildCommand": "yarn install",
+  "tests": [
+    {
+      "testName": "Example Test (Should Succeed)",
+      "testCommand": "yarn start:good",
+      "timeoutSeconds": 60
+    },
+    {
+      "testName": "Example Test (Should Fail)",
+      "testCommand": "yarn start:bad"
+    },
+    {
+      "testName": "Example Test (Should time out)",
+      "testCommand": "yarn start:timeout",
+      "timeoutSeconds": 5
+    }
+  ]
+}
diff --git a/packages/e2e-tests/test-applications/temporary-app-2/timeout.js b/packages/e2e-tests/test-applications/temporary-app-2/timeout.js
new file mode 100644
index 000000000000..044ab5d7191a
--- /dev/null
+++ b/packages/e2e-tests/test-applications/temporary-app-2/timeout.js
@@ -0,0 +1,3 @@
+setTimeout(() => {
+  console.log('Bored :/');
+}, 6000);
diff --git a/packages/e2e-tests/test-recipe-schema.json b/packages/e2e-tests/test-recipe-schema.json
new file mode 100644
index 000000000000..f2fd9a25659b
--- /dev/null
+++ b/packages/e2e-tests/test-recipe-schema.json
@@ -0,0 +1,38 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "Sentry JavaScript E2E Test Recipe",
+  "type": "object",
+  "properties": {
+    "testApplicationName": {
+      "type": "string",
+      "description": "Name displayed in test output"
+    },
+    "buildCommand": {
+      "type": "string",
+      "description": "Command that is run to install dependencies and build the test application. This command is only run once before all tests. Working directory of the command is the root of the test application."
+    },
+    "tests": {
+      "type": "array",
+      "description": "Tests to run in this test application",
+      "items": {
+        "type": "object",
+        "properties": {
+          "testName": {
+            "type": "string",
+            "description": "Name displayed in test output"
+          },
+          "testCommand": {
+            "type": "string",
+            "description": "Command that is run to start the test. Working directory of the command is the root of the test application. If this command returns a non-zero exit code the test counts as failed."
+          },
+          "timeoutSeconds": {
+            "type": "number",
+            "description": "Test timeout in seconds. Default: 60"
+          }
+        },
+        "required": ["testName", "testCommand"]
+      }
+    }
+  },
+  "required": ["testApplicationName", "tests"]
+}

From 569f85d68f84ac6fc092ff7c95a4f2d49ce6dbde Mon Sep 17 00:00:00 2001
From: Luca Forstner <luca.forstner@sentry.io>
Date: Wed, 28 Sep 2022 11:56:23 +0000
Subject: [PATCH 03/14] Exit on failure of commands

---
 packages/e2e-tests/run.ts | 34 +++++++++++++++++++++++-----------
 1 file changed, 23 insertions(+), 11 deletions(-)

diff --git a/packages/e2e-tests/run.ts b/packages/e2e-tests/run.ts
index 2acaee570f23..e0e290f38d86 100644
--- a/packages/e2e-tests/run.ts
+++ b/packages/e2e-tests/run.ts
@@ -36,16 +36,12 @@ function printCIErrorMessage(message: string): void {
 }
 
 groupCIOutput('Test Registry Setup', () => {
-  try {
-    // Stop test registry container (Verdaccio) if it was already running
-    childProcess.spawnSync('docker', ['stop', TEST_REGISTRY_CONTAINER_NAME], { stdio: 'ignore' });
-    console.log('Stopped previously running test registry');
-  } catch (e) {
-    // Don't throw if container wasn't running
-  }
+  // Stop test registry container (Verdaccio) if it was already running
+  childProcess.spawnSync('docker', ['stop', TEST_REGISTRY_CONTAINER_NAME], { stdio: 'ignore' });
+  console.log('Stopped previously running test registry');
 
   // Start test registry (Verdaccio)
-  childProcess.spawnSync(
+  const startRegistryProcessResult = childProcess.spawnSync(
     'docker',
     [
       'run',
@@ -62,8 +58,12 @@ groupCIOutput('Test Registry Setup', () => {
     { encoding: 'utf8', stdio: 'inherit' },
   );
 
+  if (startRegistryProcessResult.status !== 0) {
+    process.exit(1);
+  }
+
   // Build container image that is uploading our packages to fake registry with specific Node.js/npm version
-  childProcess.spawnSync(
+  const buildPublishImageProcessResult = childProcess.spawnSync(
     'docker',
     [
       'build',
@@ -80,8 +80,12 @@ groupCIOutput('Test Registry Setup', () => {
     },
   );
 
+  if (buildPublishImageProcessResult.status !== 0) {
+    process.exit(1);
+  }
+
   // Run container that uploads our packages to fake registry
-  childProcess.spawnSync(
+  const publishImageContainerRunProcess = childProcess.spawnSync(
     'docker',
     [
       'run',
@@ -97,6 +101,10 @@ groupCIOutput('Test Registry Setup', () => {
       stdio: 'inherit',
     },
   );
+
+  if (publishImageContainerRunProcess.status !== 0) {
+    process.exit(1);
+  }
 });
 
 groupCIOutput('Run E2E Test Suites', () => {
@@ -119,11 +127,15 @@ groupCIOutput('Run E2E Test Suites', () => {
     if (recipe.buildCommand) {
       console.log(`Running E2E test build command for test application "${recipe.testApplicationName}"`);
       const [buildCommand, ...buildCommandArgs] = recipe.buildCommand.split(' ');
-      childProcess.spawnSync(buildCommand, buildCommandArgs, {
+      const buildCommandProcess = childProcess.spawnSync(buildCommand, buildCommandArgs, {
         cwd: path.dirname(recipePath),
         encoding: 'utf8',
         stdio: 'inherit',
       });
+
+      if (buildCommandProcess.status !== 0) {
+        process.exit(1);
+      }
     }
 
     type TestResult = {

From 78b8ccebc33b72e6c5ea26cd32160554b3a683dc Mon Sep 17 00:00:00 2001
From: Luca Forstner <luca.forstner@sentry.io>
Date: Wed, 28 Sep 2022 11:59:26 +0000
Subject: [PATCH 04/14] Crash on failed tests

---
 packages/e2e-tests/run.ts | 194 ++++++++++++++++++++------------------
 1 file changed, 101 insertions(+), 93 deletions(-)

diff --git a/packages/e2e-tests/run.ts b/packages/e2e-tests/run.ts
index e0e290f38d86..20ed15c9594f 100644
--- a/packages/e2e-tests/run.ts
+++ b/packages/e2e-tests/run.ts
@@ -107,108 +107,109 @@ groupCIOutput('Test Registry Setup', () => {
   }
 });
 
-groupCIOutput('Run E2E Test Suites', () => {
-  // TODO: Run e2e tests here
-  const recipePaths = glob.sync(`${__dirname}/test-applications/*/test-recipe.json`, { absolute: true });
-
-  const recipeResults = recipePaths.map(recipePath => {
-    type Recipe = {
-      testApplicationName: string;
-      buildCommand?: string;
-      tests: {
-        testName: string;
-        testCommand: string;
-        timeoutSeconds?: number;
-      }[];
-    };
-
-    const recipe: Recipe = JSON.parse(fs.readFileSync(recipePath, 'utf-8'));
-
-    if (recipe.buildCommand) {
-      console.log(`Running E2E test build command for test application "${recipe.testApplicationName}"`);
-      const [buildCommand, ...buildCommandArgs] = recipe.buildCommand.split(' ');
-      const buildCommandProcess = childProcess.spawnSync(buildCommand, buildCommandArgs, {
-        cwd: path.dirname(recipePath),
-        encoding: 'utf8',
-        stdio: 'inherit',
-      });
-
-      if (buildCommandProcess.status !== 0) {
-        process.exit(1);
-      }
-    }
+const recipePaths = glob.sync(`${__dirname}/test-applications/*/test-recipe.json`, { absolute: true });
+
+let someTestFailed = false;
 
-    type TestResult = {
+const recipeResults = recipePaths.map(recipePath => {
+  type Recipe = {
+    testApplicationName: string;
+    buildCommand?: string;
+    tests: {
       testName: string;
-      result: 'PASS' | 'FAIL' | 'TIMEOUT';
-    };
+      testCommand: string;
+      timeoutSeconds?: number;
+    }[];
+  };
+
+  const recipe: Recipe = JSON.parse(fs.readFileSync(recipePath, 'utf-8'));
+
+  if (recipe.buildCommand) {
+    console.log(`Running E2E test build command for test application "${recipe.testApplicationName}"`);
+    const [buildCommand, ...buildCommandArgs] = recipe.buildCommand.split(' ');
+    const buildCommandProcess = childProcess.spawnSync(buildCommand, buildCommandArgs, {
+      cwd: path.dirname(recipePath),
+      encoding: 'utf8',
+      stdio: 'inherit',
+    });
 
-    const testResults: TestResult[] = recipe.tests.map(test => {
-      console.log(
-        `Running E2E test command for test application "${recipe.testApplicationName}", test "${test.testName}"`,
-      );
+    if (buildCommandProcess.status !== 0) {
+      process.exit(1);
+    }
+  }
 
-      const [testCommand, ...testCommandArgs] = test.testCommand.split(' ');
-      const testProcessResult = childProcess.spawnSync(testCommand, testCommandArgs, {
-        cwd: path.dirname(recipePath),
-        timeout: (test.timeoutSeconds ?? DEFAULT_TEST_TIMEOUT_SECONDS) * 1000,
-        encoding: 'utf8',
-        stdio: 'pipe',
-      });
-
-      console.log(testProcessResult.stdout.replace(/^/gm, '[TEST OUTPUT] '));
-      console.log(testProcessResult.stderr.replace(/^/gm, '[TEST OUTPUT] '));
-
-      const error: undefined | (Error & { code?: string }) = testProcessResult.error;
-
-      if (error?.code === 'ETIMEDOUT') {
-        printCIErrorMessage(
-          `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname(
-            recipePath,
-          )}) timed out.`,
-        );
-        return {
-          testName: test.testName,
-          result: 'TIMEOUT',
-        };
-      } else if (testProcessResult.status !== 0) {
-        printCIErrorMessage(
-          `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname(
-            recipePath,
-          )}) failed.`,
-        );
-        return {
-          testName: test.testName,
-          result: 'FAIL',
-        };
-      } else {
-        console.log(
-          `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname(
-            recipePath,
-          )}) succeeded.`,
-        );
-        return {
-          testName: test.testName,
-          result: 'PASS',
-        };
-      }
+  type TestResult = {
+    testName: string;
+    result: 'PASS' | 'FAIL' | 'TIMEOUT';
+  };
+
+  const testResults: TestResult[] = recipe.tests.map(test => {
+    console.log(
+      `Running E2E test command for test application "${recipe.testApplicationName}", test "${test.testName}"`,
+    );
+
+    const [testCommand, ...testCommandArgs] = test.testCommand.split(' ');
+    const testProcessResult = childProcess.spawnSync(testCommand, testCommandArgs, {
+      cwd: path.dirname(recipePath),
+      timeout: (test.timeoutSeconds ?? DEFAULT_TEST_TIMEOUT_SECONDS) * 1000,
+      encoding: 'utf8',
+      stdio: 'pipe',
     });
 
-    return {
-      testApplicationName: recipe.testApplicationName,
-      testApplicationPath: recipePath,
-      testResults,
-    };
+    console.log(testProcessResult.stdout.replace(/^/gm, '[TEST OUTPUT] '));
+    console.log(testProcessResult.stderr.replace(/^/gm, '[TEST OUTPUT] '));
+
+    const error: undefined | (Error & { code?: string }) = testProcessResult.error;
+
+    if (error?.code === 'ETIMEDOUT') {
+      printCIErrorMessage(
+        `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname(
+          recipePath,
+        )}) timed out.`,
+      );
+      return {
+        testName: test.testName,
+        result: 'TIMEOUT',
+      };
+    } else if (testProcessResult.status !== 0) {
+      someTestFailed = true;
+      printCIErrorMessage(
+        `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname(
+          recipePath,
+        )}) failed.`,
+      );
+      return {
+        testName: test.testName,
+        result: 'FAIL',
+      };
+    } else {
+      someTestFailed = true;
+      console.log(
+        `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname(
+          recipePath,
+        )}) succeeded.`,
+      );
+      return {
+        testName: test.testName,
+        result: 'PASS',
+      };
+    }
   });
 
-  console.log('--------------------------------------');
-  console.log('Test Result Summary:');
+  return {
+    testApplicationName: recipe.testApplicationName,
+    testApplicationPath: recipePath,
+    testResults,
+  };
+});
 
-  recipeResults.forEach(recipeResult => {
-    console.log(`● ${recipeResult.testApplicationName} (${path.dirname(recipeResult.testApplicationPath)})`);
-    recipeResult.testResults.forEach(testResult => {
-      console.log(`  ● ${testResult.result.padEnd(7, ' ')} ${testResult.testName}`);
-    });
+console.log('--------------------------------------');
+console.log('Test Result Summary:');
+
+recipeResults.forEach(recipeResult => {
+  console.log(`● ${recipeResult.testApplicationName} (${path.dirname(recipeResult.testApplicationPath)})`);
+  recipeResult.testResults.forEach(testResult => {
+    console.log(`  ● ${testResult.result.padEnd(7, ' ')} ${testResult.testName}`);
   });
 });
 
@@ -217,3 +218,10 @@ groupCIOutput('Cleanup', () => {
   childProcess.spawnSync(`docker stop ${TEST_REGISTRY_CONTAINER_NAME}`, { encoding: 'utf8', stdio: 'ignore' });
   console.log('Successfully stopped test registry container'); // Output from command above is not good so we `ignore` it and emit our own
 });
+
+if (someTestFailed) {
+  console.log('Not all tests succeeded.');
+  process.exit(1);
+} else {
+  console.log('All tests succeeded.');
+}

From d0709fe9a47f0b74681fa5d4586e7d209c3c6367 Mon Sep 17 00:00:00 2001
From: Luca Forstner <luca.forstner@sentry.io>
Date: Wed, 28 Sep 2022 12:03:25 +0000
Subject: [PATCH 05/14] No arg splitting

---
 packages/e2e-tests/run.ts | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/packages/e2e-tests/run.ts b/packages/e2e-tests/run.ts
index 20ed15c9594f..d7e93cfcc6c4 100644
--- a/packages/e2e-tests/run.ts
+++ b/packages/e2e-tests/run.ts
@@ -126,11 +126,11 @@ const recipeResults = recipePaths.map(recipePath => {
 
   if (recipe.buildCommand) {
     console.log(`Running E2E test build command for test application "${recipe.testApplicationName}"`);
-    const [buildCommand, ...buildCommandArgs] = recipe.buildCommand.split(' ');
-    const buildCommandProcess = childProcess.spawnSync(buildCommand, buildCommandArgs, {
+    const buildCommandProcess = childProcess.spawnSync(recipe.buildCommand, {
       cwd: path.dirname(recipePath),
       encoding: 'utf8',
       stdio: 'inherit',
+      shell: true, // needed so we can pass the build command in as whole without splitting it up into args
     });
 
     if (buildCommandProcess.status !== 0) {
@@ -148,12 +148,12 @@ const recipeResults = recipePaths.map(recipePath => {
       `Running E2E test command for test application "${recipe.testApplicationName}", test "${test.testName}"`,
     );
 
-    const [testCommand, ...testCommandArgs] = test.testCommand.split(' ');
-    const testProcessResult = childProcess.spawnSync(testCommand, testCommandArgs, {
+    const testProcessResult = childProcess.spawnSync(test.testCommand, {
       cwd: path.dirname(recipePath),
       timeout: (test.timeoutSeconds ?? DEFAULT_TEST_TIMEOUT_SECONDS) * 1000,
       encoding: 'utf8',
       stdio: 'pipe',
+      shell: true, // needed so we can pass the test command in as whole without splitting it up into args
     });
 
     console.log(testProcessResult.stdout.replace(/^/gm, '[TEST OUTPUT] '));

From d46c984a64cdad843546eba07890c746274603ff Mon Sep 17 00:00:00 2001
From: Luca Forstner <luca.forstner@sentry.io>
Date: Wed, 28 Sep 2022 12:56:20 +0000
Subject: [PATCH 06/14] Fix build args

---
 packages/e2e-tests/run.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/e2e-tests/run.ts b/packages/e2e-tests/run.ts
index d7e93cfcc6c4..e48e6135f701 100644
--- a/packages/e2e-tests/run.ts
+++ b/packages/e2e-tests/run.ts
@@ -71,9 +71,9 @@ groupCIOutput('Test Registry Setup', () => {
       PUBLISH_PACKAGES_DOCKER_IMAGE_NAME,
       '--file',
       './Dockerfile.publish-packages',
-      publishScriptNodeVersion ? `--build-arg NODE_VERSION=${publishScriptNodeVersion}` : undefined,
+      ...(publishScriptNodeVersion ? ['--build-arg', `NODE_VERSION=${publishScriptNodeVersion}`] : []),
       '.',
-    ].filter((arg): arg is string => arg !== undefined),
+    ],
     {
       encoding: 'utf8',
       stdio: 'inherit',

From 6bce0de34e143d5b9518a968093345300e4696b2 Mon Sep 17 00:00:00 2001
From: Luca Forstner <luca.forstner@sentry.io>
Date: Wed, 28 Sep 2022 13:33:14 +0000
Subject: [PATCH 07/14] Make E2E tests pass so we don't fail master

---
 packages/e2e-tests/run.ts                            |  9 ++++++---
 .../test-applications/temporary-app-1/bad.js         |  1 -
 .../temporary-app-1/test-recipe.json                 | 12 +-----------
 .../test-applications/temporary-app-1/timeout.js     |  3 ---
 .../test-applications/temporary-app-2/bad.js         |  1 -
 .../temporary-app-2/test-recipe.json                 | 12 +-----------
 .../test-applications/temporary-app-2/timeout.js     |  3 ---
 7 files changed, 8 insertions(+), 33 deletions(-)
 delete mode 100644 packages/e2e-tests/test-applications/temporary-app-1/bad.js
 delete mode 100644 packages/e2e-tests/test-applications/temporary-app-1/timeout.js
 delete mode 100644 packages/e2e-tests/test-applications/temporary-app-2/bad.js
 delete mode 100644 packages/e2e-tests/test-applications/temporary-app-2/timeout.js

diff --git a/packages/e2e-tests/run.ts b/packages/e2e-tests/run.ts
index e48e6135f701..aa26c774c54e 100644
--- a/packages/e2e-tests/run.ts
+++ b/packages/e2e-tests/run.ts
@@ -129,10 +129,13 @@ const recipeResults = recipePaths.map(recipePath => {
     const buildCommandProcess = childProcess.spawnSync(recipe.buildCommand, {
       cwd: path.dirname(recipePath),
       encoding: 'utf8',
-      stdio: 'inherit',
       shell: true, // needed so we can pass the build command in as whole without splitting it up into args
     });
 
+    // Prepends some text to the output build command's output so we can distinguish it from logging in this script
+    console.log(buildCommandProcess.stdout.replace(/^/gm, '[BUILD OUTPUT] '));
+    console.log(buildCommandProcess.stderr.replace(/^/gm, '[BUILD OUTPUT] '));
+
     if (buildCommandProcess.status !== 0) {
       process.exit(1);
     }
@@ -152,16 +155,17 @@ const recipeResults = recipePaths.map(recipePath => {
       cwd: path.dirname(recipePath),
       timeout: (test.timeoutSeconds ?? DEFAULT_TEST_TIMEOUT_SECONDS) * 1000,
       encoding: 'utf8',
-      stdio: 'pipe',
       shell: true, // needed so we can pass the test command in as whole without splitting it up into args
     });
 
+    // Prepends some text to the output test command's output so we can distinguish it from logging in this script
     console.log(testProcessResult.stdout.replace(/^/gm, '[TEST OUTPUT] '));
     console.log(testProcessResult.stderr.replace(/^/gm, '[TEST OUTPUT] '));
 
     const error: undefined | (Error & { code?: string }) = testProcessResult.error;
 
     if (error?.code === 'ETIMEDOUT') {
+      someTestFailed = true;
       printCIErrorMessage(
         `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname(
           recipePath,
@@ -183,7 +187,6 @@ const recipeResults = recipePaths.map(recipePath => {
         result: 'FAIL',
       };
     } else {
-      someTestFailed = true;
       console.log(
         `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname(
           recipePath,
diff --git a/packages/e2e-tests/test-applications/temporary-app-1/bad.js b/packages/e2e-tests/test-applications/temporary-app-1/bad.js
deleted file mode 100644
index bf99992592d6..000000000000
--- a/packages/e2e-tests/test-applications/temporary-app-1/bad.js
+++ /dev/null
@@ -1 +0,0 @@
-throw new Error('Sad :(');
diff --git a/packages/e2e-tests/test-applications/temporary-app-1/test-recipe.json b/packages/e2e-tests/test-applications/temporary-app-1/test-recipe.json
index ddc1f2e72d9e..480f0ce4003e 100644
--- a/packages/e2e-tests/test-applications/temporary-app-1/test-recipe.json
+++ b/packages/e2e-tests/test-applications/temporary-app-1/test-recipe.json
@@ -5,17 +5,7 @@
   "tests": [
     {
       "testName": "Example Test (Should Succeed)",
-      "testCommand": "yarn start:good",
-      "timeoutSeconds": 30
-    },
-    {
-      "testName": "Example Test (Should Fail)",
-      "testCommand": "yarn start:bad"
-    },
-    {
-      "testName": "Example Test (Should time out)",
-      "testCommand": "yarn start:timeout",
-      "timeoutSeconds": 5
+      "testCommand": "yarn start:good"
     }
   ]
 }
diff --git a/packages/e2e-tests/test-applications/temporary-app-1/timeout.js b/packages/e2e-tests/test-applications/temporary-app-1/timeout.js
deleted file mode 100644
index 044ab5d7191a..000000000000
--- a/packages/e2e-tests/test-applications/temporary-app-1/timeout.js
+++ /dev/null
@@ -1,3 +0,0 @@
-setTimeout(() => {
-  console.log('Bored :/');
-}, 6000);
diff --git a/packages/e2e-tests/test-applications/temporary-app-2/bad.js b/packages/e2e-tests/test-applications/temporary-app-2/bad.js
deleted file mode 100644
index bf99992592d6..000000000000
--- a/packages/e2e-tests/test-applications/temporary-app-2/bad.js
+++ /dev/null
@@ -1 +0,0 @@
-throw new Error('Sad :(');
diff --git a/packages/e2e-tests/test-applications/temporary-app-2/test-recipe.json b/packages/e2e-tests/test-applications/temporary-app-2/test-recipe.json
index 9d93f49cc667..c9367aa534ea 100644
--- a/packages/e2e-tests/test-applications/temporary-app-2/test-recipe.json
+++ b/packages/e2e-tests/test-applications/temporary-app-2/test-recipe.json
@@ -5,17 +5,7 @@
   "tests": [
     {
       "testName": "Example Test (Should Succeed)",
-      "testCommand": "yarn start:good",
-      "timeoutSeconds": 60
-    },
-    {
-      "testName": "Example Test (Should Fail)",
-      "testCommand": "yarn start:bad"
-    },
-    {
-      "testName": "Example Test (Should time out)",
-      "testCommand": "yarn start:timeout",
-      "timeoutSeconds": 5
+      "testCommand": "yarn start:good"
     }
   ]
 }
diff --git a/packages/e2e-tests/test-applications/temporary-app-2/timeout.js b/packages/e2e-tests/test-applications/temporary-app-2/timeout.js
deleted file mode 100644
index 044ab5d7191a..000000000000
--- a/packages/e2e-tests/test-applications/temporary-app-2/timeout.js
+++ /dev/null
@@ -1,3 +0,0 @@
-setTimeout(() => {
-  console.log('Bored :/');
-}, 6000);

From 0390c53318621c2ed549cacbbb8eded9183b14dc Mon Sep 17 00:00:00 2001
From: Luca Forstner <luca.forstner@sentry.io>
Date: Wed, 28 Sep 2022 14:32:11 +0000
Subject: [PATCH 08/14] Improve failed build reporting

---
 packages/e2e-tests/run.ts | 57 ++++++++++++++++++++++++++++-----------
 1 file changed, 42 insertions(+), 15 deletions(-)

diff --git a/packages/e2e-tests/run.ts b/packages/e2e-tests/run.ts
index aa26c774c54e..0e26418b66a7 100644
--- a/packages/e2e-tests/run.ts
+++ b/packages/e2e-tests/run.ts
@@ -109,9 +109,21 @@ groupCIOutput('Test Registry Setup', () => {
 
 const recipePaths = glob.sync(`${__dirname}/test-applications/*/test-recipe.json`, { absolute: true });
 
-let someTestFailed = false;
+let processShouldExitWithError = false;
 
-const recipeResults = recipePaths.map(recipePath => {
+type TestResult = {
+  testName: string;
+  result: 'PASS' | 'FAIL' | 'TIMEOUT';
+};
+
+type RecipeResult = {
+  testApplicationName: string;
+  testApplicationPath: string;
+  buildFailed: boolean;
+  testResults: TestResult[];
+};
+
+const recipeResults: RecipeResult[] = recipePaths.map(recipePath => {
   type Recipe = {
     testApplicationName: string;
     buildCommand?: string;
@@ -137,15 +149,21 @@ const recipeResults = recipePaths.map(recipePath => {
     console.log(buildCommandProcess.stderr.replace(/^/gm, '[BUILD OUTPUT] '));
 
     if (buildCommandProcess.status !== 0) {
-      process.exit(1);
+      processShouldExitWithError = true;
+
+      printCIErrorMessage(
+        `Build command in test application "${recipe.testApplicationName}" (${path.dirname(recipePath)}) failed!`,
+      );
+
+      return {
+        testApplicationName: recipe.testApplicationName,
+        testApplicationPath: recipePath,
+        buildFailed: true,
+        testResults: [],
+      };
     }
   }
 
-  type TestResult = {
-    testName: string;
-    result: 'PASS' | 'FAIL' | 'TIMEOUT';
-  };
-
   const testResults: TestResult[] = recipe.tests.map(test => {
     console.log(
       `Running E2E test command for test application "${recipe.testApplicationName}", test "${test.testName}"`,
@@ -165,7 +183,7 @@ const recipeResults = recipePaths.map(recipePath => {
     const error: undefined | (Error & { code?: string }) = testProcessResult.error;
 
     if (error?.code === 'ETIMEDOUT') {
-      someTestFailed = true;
+      processShouldExitWithError = true;
       printCIErrorMessage(
         `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname(
           recipePath,
@@ -176,7 +194,7 @@ const recipeResults = recipePaths.map(recipePath => {
         result: 'TIMEOUT',
       };
     } else if (testProcessResult.status !== 0) {
-      someTestFailed = true;
+      processShouldExitWithError = true;
       printCIErrorMessage(
         `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname(
           recipePath,
@@ -202,6 +220,7 @@ const recipeResults = recipePaths.map(recipePath => {
   return {
     testApplicationName: recipe.testApplicationName,
     testApplicationPath: recipePath,
+    buildFailed: false,
     testResults,
   };
 });
@@ -210,10 +229,18 @@ console.log('--------------------------------------');
 console.log('Test Result Summary:');
 
 recipeResults.forEach(recipeResult => {
-  console.log(`● ${recipeResult.testApplicationName} (${path.dirname(recipeResult.testApplicationPath)})`);
-  recipeResult.testResults.forEach(testResult => {
-    console.log(`  ● ${testResult.result.padEnd(7, ' ')} ${testResult.testName}`);
-  });
+  if (recipeResult.buildFailed) {
+    console.log(
+      `● BUILD FAILED - ${recipeResult.testApplicationName} (${path.dirname(recipeResult.testApplicationPath)})`,
+    );
+  } else {
+    console.log(
+      `● BUILD SUCCEEDED - ${recipeResult.testApplicationName} (${path.dirname(recipeResult.testApplicationPath)})`,
+    );
+    recipeResult.testResults.forEach(testResult => {
+      console.log(`  ● ${testResult.result.padEnd(7, ' ')} ${testResult.testName}`);
+    });
+  }
 });
 
 groupCIOutput('Cleanup', () => {
@@ -222,7 +249,7 @@ groupCIOutput('Cleanup', () => {
   console.log('Successfully stopped test registry container'); // Output from command above is not good so we `ignore` it and emit our own
 });
 
-if (someTestFailed) {
+if (processShouldExitWithError) {
   console.log('Not all tests succeeded.');
   process.exit(1);
 } else {

From 7995d93792b74f7cea9193d17a122be13709f88b Mon Sep 17 00:00:00 2001
From: Luca Forstner <luca.forstner@sentry.io>
Date: Wed, 28 Sep 2022 14:32:42 +0000
Subject: [PATCH 09/14] Remove unnecessary scripts

---
 .../e2e-tests/test-applications/temporary-app-1/package.json  | 4 +---
 .../e2e-tests/test-applications/temporary-app-2/package.json  | 4 +---
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/packages/e2e-tests/test-applications/temporary-app-1/package.json b/packages/e2e-tests/test-applications/temporary-app-1/package.json
index 449dbd698c88..6adffeacc75f 100644
--- a/packages/e2e-tests/test-applications/temporary-app-1/package.json
+++ b/packages/e2e-tests/test-applications/temporary-app-1/package.json
@@ -3,9 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "scripts": {
-    "start:good": "node good.js",
-    "start:bad": "node bad.js",
-    "start:timeout": "node timeout.js"
+    "start:good": "node good.js"
   },
   "dependencies": {
     "@sentry/node": "*"
diff --git a/packages/e2e-tests/test-applications/temporary-app-2/package.json b/packages/e2e-tests/test-applications/temporary-app-2/package.json
index b9a5cb6f68e7..a01456cb2f2e 100644
--- a/packages/e2e-tests/test-applications/temporary-app-2/package.json
+++ b/packages/e2e-tests/test-applications/temporary-app-2/package.json
@@ -3,9 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "scripts": {
-    "start:good": "node good.js",
-    "start:bad": "node bad.js",
-    "start:timeout": "node timeout.js"
+    "start:good": "node good.js"
   },
   "dependencies": {
     "@sentry/node": "*"

From fbeafabd296f796b9002a9683f35b7a34e627e78 Mon Sep 17 00:00:00 2001
From: Luca Forstner <luca.forstner@sentry.io>
Date: Wed, 28 Sep 2022 15:00:39 +0000
Subject: [PATCH 10/14] Remove unnecessary configuration

---
 packages/e2e-tests/test-applications/temporary-app-1/.npmrc | 1 -
 packages/e2e-tests/test-applications/temporary-app-2/.npmrc | 1 -
 2 files changed, 2 deletions(-)

diff --git a/packages/e2e-tests/test-applications/temporary-app-1/.npmrc b/packages/e2e-tests/test-applications/temporary-app-1/.npmrc
index c35d987cca9f..c6b3ef9b3eaa 100644
--- a/packages/e2e-tests/test-applications/temporary-app-1/.npmrc
+++ b/packages/e2e-tests/test-applications/temporary-app-1/.npmrc
@@ -1,3 +1,2 @@
 @sentry:registry=http://localhost:4873
 @sentry-internal:registry=http://localhost:4873
-//localhost:4873/:_authToken=some-token
diff --git a/packages/e2e-tests/test-applications/temporary-app-2/.npmrc b/packages/e2e-tests/test-applications/temporary-app-2/.npmrc
index c35d987cca9f..c6b3ef9b3eaa 100644
--- a/packages/e2e-tests/test-applications/temporary-app-2/.npmrc
+++ b/packages/e2e-tests/test-applications/temporary-app-2/.npmrc
@@ -1,3 +1,2 @@
 @sentry:registry=http://localhost:4873
 @sentry-internal:registry=http://localhost:4873
-//localhost:4873/:_authToken=some-token

From f9800015a286b52d8fc886eb4dbb861ab37ca518 Mon Sep 17 00:00:00 2001
From: Luca Forstner <luca.forstner@sentry.io>
Date: Wed, 28 Sep 2022 15:06:05 +0000
Subject: [PATCH 11/14] Don't create lockfiles

---
 .../test-applications/temporary-app-1/test-recipe.json          | 2 +-
 .../test-applications/temporary-app-2/test-recipe.json          | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/e2e-tests/test-applications/temporary-app-1/test-recipe.json b/packages/e2e-tests/test-applications/temporary-app-1/test-recipe.json
index 480f0ce4003e..c00b1ff55a7e 100644
--- a/packages/e2e-tests/test-applications/temporary-app-1/test-recipe.json
+++ b/packages/e2e-tests/test-applications/temporary-app-1/test-recipe.json
@@ -1,7 +1,7 @@
 {
   "$schema": "../../test-recipe-schema.json",
   "testApplicationName": "Temporary Application 1",
-  "buildCommand": "yarn install",
+  "buildCommand": "yarn install --no-lockfile",
   "tests": [
     {
       "testName": "Example Test (Should Succeed)",
diff --git a/packages/e2e-tests/test-applications/temporary-app-2/test-recipe.json b/packages/e2e-tests/test-applications/temporary-app-2/test-recipe.json
index c9367aa534ea..2ccdd52d62fb 100644
--- a/packages/e2e-tests/test-applications/temporary-app-2/test-recipe.json
+++ b/packages/e2e-tests/test-applications/temporary-app-2/test-recipe.json
@@ -1,7 +1,7 @@
 {
   "$schema": "../../test-recipe-schema.json",
   "testApplicationName": "Temporary Application 2",
-  "buildCommand": "yarn install",
+  "buildCommand": "yarn install --no-lockfile",
   "tests": [
     {
       "testName": "Example Test (Should Succeed)",

From d5a5694ca0d5d1772161de3bf9d53597ef41d53f Mon Sep 17 00:00:00 2001
From: Luca Forstner <luca.forstner@sentry.io>
Date: Wed, 28 Sep 2022 15:06:38 +0000
Subject: [PATCH 12/14] Add readme

---
 packages/e2e-tests/README.md | 91 ++++++++++++++++++++++++++++++++++++
 1 file changed, 91 insertions(+)
 create mode 100644 packages/e2e-tests/README.md

diff --git a/packages/e2e-tests/README.md b/packages/e2e-tests/README.md
new file mode 100644
index 000000000000..18babe21327c
--- /dev/null
+++ b/packages/e2e-tests/README.md
@@ -0,0 +1,91 @@
+# E2E Tests
+
+E2E tests enable us to verify the behavior of the packages in this repository as if they were to be published in their current state.
+
+## How to run
+
+Prerequisites: Docker
+
+```bash
+yarn test:e2e
+```
+
+## How they work
+
+Before running any tests we launch a fake test registry (in our case [Verdaccio](https://verdaccio.org/docs/e2e/)), we build our packages, pack them, and publish them to the fake registry.
+The fake registry is hosted in a Docker container, and the script to publish the packages is also run from within a container to ensure that the fake publishing happens with the same Node.js and npm versions as we're using in CI.
+
+After publishing our freshly built packages to the fake registry, the E2E test script will look for `test-recipe.json` files in test applications located in the `test-applications` folder.
+In this folder, we keep standalone test applications, that use our SDKs and can be used to verify their behavior.
+The `test-recipe.json` recipe files contain information on how to build the test applications and how to run tests on these applications.
+
+## How to set up a new test
+
+Test applications are completely standalone applications that can be used to verify our SDKs.
+To set one up, follow these commands:
+
+```sh
+cd packages/e2e-tests
+
+# Create a new test application folder
+mkdir test-applications/my-new-test-application # Name of the new folder doesn't technically matter but choose something meaningful
+
+# Create an npm configuration file that uses the fake test registry
+cat > test-applications/my-new-test-application/.npmrc << EOF
+@sentry:registry=http://localhost:4873
+@sentry-internal:registry=http://localhost:4873
+EOF
+
+# Add a gitignore that ignores lockfiles
+cat > test-applications/my-new-test-application/.gitignore << EOF
+yarn.lock
+package-lock.json
+EOF
+
+# Add a test recipe file to the test application
+touch test-applications/my-new-test-application/test-recipe.json
+```
+
+To get you started with the recipe, you can copy the following into `test-recipe.json`:
+
+```json
+{
+  "$schema": "../../test-recipe-schema.json",
+  "testApplicationName": "My New Test Application",
+  "buildCommand": "yarn install --no-lockfile",
+  "tests": [
+    {
+      "testName": "My new test",
+      "testCommand": "yarn test",
+      "timeoutSeconds": 60
+    }
+  ]
+}
+```
+
+The `test-recipe.json` files follow a schema (`e2e-tests/test-recipe-schema.json`). Here is a basic explanation of the fields:
+
+- The `buildCommand` command runs only once before any of the tests and is supposed to build the test application. If this command returns a non-zero exit code, it counts as a failed test and the test application's tests are not run.
+- The `testCommand` command is supposed to run tests on the test application. If the configured command returns a non-zero exit code, it counts as a failed test.
+- A test timeout can be configured via `timeoutSeconds`, it defaults to `60`.
+
+**An important thing to note:** In the context of the `buildCommand` the fake test registry is available at `http://localhost:4873`. It hosts all of our packages as if they were to be published with the state of the current branch.
+This means we can install the packages from this registry via the `.npmrc` configuration as seen above.
+If you add Sentry dependencies to your test application, you should set the dependency versions set to `*`:
+
+```jsonc
+// package.json
+{
+  "name": "my-new-test-application",
+  "version": "1.0.0",
+  "private": true,
+  "scripts": {
+    "test": "echo \"Hello world!\""
+  },
+  "dependencies": {
+    "@sentry/node": "*"
+  }
+}
+```
+
+All that is left for you to do now is to create a test app and run `yarn test:e2e`.

From abe422557e07a561ccd43f13b23ada472d6639bf Mon Sep 17 00:00:00 2001
From: Luca Forstner <luca.forstner@sentry.io>
Date: Wed, 28 Sep 2022 15:25:33 +0000
Subject: [PATCH 13/14] Run prettier

---
 packages/e2e-tests/README.md | 36 ++++++++++++++++++++++--------------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/packages/e2e-tests/README.md b/packages/e2e-tests/README.md
index 18babe21327c..fbf819dc6310 100644
--- a/packages/e2e-tests/README.md
+++ b/packages/e2e-tests/README.md
@@ -1,6 +1,7 @@
 # E2E Tests
 
-E2E tests enable us to verify the behavior of the packages in this repository as if they were to be published in their current state.
+E2E tests enable us to verify the behavior of the packages in this repository as if they were to be published in their
+current state.
 
 ## How to run
 
@@ -12,17 +13,20 @@ yarn test:e2e
 
 ## How they work
 
-Before running any tests we launch a fake test registry (in our case [Verdaccio](https://verdaccio.org/docs/e2e/)), we build our packages, pack them, and publish them to the fake registry.
-The fake registry is hosted in a Docker container, and the script to publish the packages is also run from within a container to ensure that the fake publishing happens with the same Node.js and npm versions as we're using in CI.
+Before running any tests we launch a fake test registry (in our case [Verdaccio](https://verdaccio.org/docs/e2e/)), we
+build our packages, pack them, and publish them to the fake registry. The fake registry is hosted in a Docker container,
+and the script to publish the packages is also run from within a container to ensure that the fake publishing happens
+with the same Node.js and npm versions as we're using in CI.
 
-After publishing our freshly built packages to the fake registry, the E2E test script will look for `test-recipe.json` files in test applications located in the `test-applications` folder.
-In this folder, we keep standalone test applications, that use our SDKs and can be used to verify their behavior.
-The `test-recipe.json` recipe files contain information on how to build the test applications and how to run tests on these applications.
+After publishing our freshly built packages to the fake registry, the E2E test script will look for `test-recipe.json`
+files in test applications located in the `test-applications` folder. In this folder, we keep standalone test
+applications, that use our SDKs and can be used to verify their behavior. The `test-recipe.json` recipe files contain
+information on how to build the test applications and how to run tests on these applications.
 
 ## How to set up a new test
 
-Test applications are completely standalone applications that can be used to verify our SDKs.
-To set one up, follow these commands:
+Test applications are completely standalone applications that can be used to verify our SDKs. To set one up, follow
+these commands:
 
 ```sh
 cd packages/e2e-tests
@@ -63,15 +67,19 @@ To get you started with the recipe, you can copy the following into `test-recipe
 }
 ```
 
-The `test-recipe.json` files follow a schema (`e2e-tests/test-recipe-schema.json`). Here is a basic explanation of the fields:
+The `test-recipe.json` files follow a schema (`e2e-tests/test-recipe-schema.json`). Here is a basic explanation of the
+fields:
 
-- The `buildCommand` command runs only once before any of the tests and is supposed to build the test application. If this command returns a non-zero exit code, it counts as a failed test and the test application's tests are not run.
-- The `testCommand` command is supposed to run tests on the test application. If the configured command returns a non-zero exit code, it counts as a failed test.
+- The `buildCommand` command runs only once before any of the tests and is supposed to build the test application. If
+  this command returns a non-zero exit code, it counts as a failed test and the test application's tests are not run.
+- The `testCommand` command is supposed to run tests on the test application. If the configured command returns a
+  non-zero exit code, it counts as a failed test.
 - A test timeout can be configured via `timeoutSeconds`, it defaults to `60`.
 
-**An important thing to note:** In the context of the `buildCommand` the fake test registry is available at `http://localhost:4873`. It hosts all of our packages as if they were to be published with the state of the current branch.
-This means we can install the packages from this registry via the `.npmrc` configuration as seen above.
-If you add Sentry dependencies to your test application, you should set the dependency versions set to `*`:
+**An important thing to note:** In the context of the `buildCommand` the fake test registry is available at
+`http://localhost:4873`. It hosts all of our packages as if they were to be published with the state of the current
+branch. This means we can install the packages from this registry via the `.npmrc` configuration as seen above. If you
+add Sentry dependencies to your test application, you should set the dependency versions set to `*`:
 
 ```jsonc
 // package.json

From 114ae0da64cbd4dc4159fe809a1ac23d646d98d7 Mon Sep 17 00:00:00 2001
From: Luca Forstner <luca.forstner@sentry.io>
Date: Wed, 28 Sep 2022 16:21:40 +0000
Subject: [PATCH 14/14] Add timeout for build command

---
 packages/e2e-tests/run.ts                  | 3 +++
 packages/e2e-tests/test-recipe-schema.json | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/packages/e2e-tests/run.ts b/packages/e2e-tests/run.ts
index 0e26418b66a7..88319cce20c2 100644
--- a/packages/e2e-tests/run.ts
+++ b/packages/e2e-tests/run.ts
@@ -13,6 +13,7 @@ const PUBLISH_PACKAGES_DOCKER_IMAGE_NAME = 'publish-packages';
 
 const publishScriptNodeVersion = process.env.E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION;
 
+const DEFAULT_BUILD_TIMEOUT_SECONDS = 60;
 const DEFAULT_TEST_TIMEOUT_SECONDS = 60;
 
 // https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines
@@ -127,6 +128,7 @@ const recipeResults: RecipeResult[] = recipePaths.map(recipePath => {
   type Recipe = {
     testApplicationName: string;
     buildCommand?: string;
+    buildTimeoutSeconds?: number;
     tests: {
       testName: string;
       testCommand: string;
@@ -142,6 +144,7 @@ const recipeResults: RecipeResult[] = recipePaths.map(recipePath => {
       cwd: path.dirname(recipePath),
       encoding: 'utf8',
       shell: true, // needed so we can pass the build command in as whole without splitting it up into args
+      timeout: (recipe.buildTimeoutSeconds ?? DEFAULT_BUILD_TIMEOUT_SECONDS) * 1000,
     });
 
     // Prepends some text to the output build command's output so we can distinguish it from logging in this script
diff --git a/packages/e2e-tests/test-recipe-schema.json b/packages/e2e-tests/test-recipe-schema.json
index f2fd9a25659b..8c0a93fe11c5 100644
--- a/packages/e2e-tests/test-recipe-schema.json
+++ b/packages/e2e-tests/test-recipe-schema.json
@@ -11,6 +11,10 @@
       "type": "string",
       "description": "Command that is run to install dependencies and build the test application. This command is only run once before all tests. Working directory of the command is the root of the test application."
     },
+    "buildTimeoutSeconds": {
+      "type": "number",
+      "description": "Timeout for the build command in seconds. Default: 60"
+    },
     "tests": {
       "type": "array",
       "description": "Tests to run in this test application",