DONT MERGE ME, I AM JUST HERE TO SHOWCASE FAILING E2E TESTS

packages/e2e-tests/.eslintrc.js

@@ -3,7 +3,7 @@ module.exports = {
     node: true,
   },
   extends: ['../../.eslintrc.js'],
-  ignorePatterns: [],
+  ignorePatterns: ['test-applications/**'],
   parserOptions: {
     sourceType: 'module',
   },

packages/e2e-tests/run.ts

@@ -1,5 +1,7 @@
 /* eslint-disable no-console */
 import * as childProcess from 'child_process';
+import * as fs from 'fs';
+import * as glob from 'glob';
 import * as path from 'path';
 
 const repositoryRoot = path.resolve(__dirname, '../..');
@@ -11,42 +13,215 @@ const PUBLISH_PACKAGES_DOCKER_IMAGE_NAME = 'publish-packages';
 
 const publishScriptNodeVersion = process.env.E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION;
 
-try {
+const DEFAULT_TEST_TIMEOUT_SECONDS = 60;
+
+// https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines
+function groupCIOutput(groupTitle: string, fn: () => void): void {
+  if (process.env.CI) {
+    console.log(`::group::${groupTitle}`);
+    fn();
+    console.log('::endgroup::');
+  } else {
+    fn();
+  }
+}
+
+// https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message
+function printCIErrorMessage(message: string): void {
+  if (process.env.CI) {
+    console.log(`::error::${message}`);
+  } else {
+    console.log(message);
+  }
+}
+
+groupCIOutput('Test Registry Setup', () => {
   // Stop test registry container (Verdaccio) if it was already running
-  childProcess.execSync(`docker stop ${TEST_REGISTRY_CONTAINER_NAME}`, { stdio: 'ignore' });
+  childProcess.spawnSync('docker', ['stop', TEST_REGISTRY_CONTAINER_NAME], { stdio: 'ignore' });
   console.log('Stopped previously running test registry');
-} catch (e) {
-  // Don't throw if container wasn't running
-}
 
-// Start test registry (Verdaccio)
-childProcess.execSync(
-  `docker run --detach --rm --name ${TEST_REGISTRY_CONTAINER_NAME} -p 4873:4873 -v ${__dirname}/verdaccio-config:/verdaccio/conf verdaccio/verdaccio:${VERDACCIO_VERSION}`,
-  { encoding: 'utf8', stdio: 'inherit' },
-);
-
-// Build container image that is uploading our packages to fake registry with specific Node.js/npm version
-childProcess.execSync(
-  `docker build --tag ${PUBLISH_PACKAGES_DOCKER_IMAGE_NAME} --file ./Dockerfile.publish-packages ${
-    publishScriptNodeVersion ? `--build-arg NODE_VERSION=${publishScriptNodeVersion}` : ''
-  } .`,
-  {
-    encoding: 'utf8',
-    stdio: 'inherit',
-  },
-);
-
-// Run container that uploads our packages to fake registry
-childProcess.execSync(
-  `docker run --rm -v ${repositoryRoot}:/sentry-javascript --network host ${PUBLISH_PACKAGES_DOCKER_IMAGE_NAME}`,
-  {
-    encoding: 'utf8',
-    stdio: 'inherit',
-  },
-);
-
-// TODO: Run e2e tests here
-
-// Stop test registry
-childProcess.execSync(`docker stop ${TEST_REGISTRY_CONTAINER_NAME}`, { encoding: 'utf8', stdio: 'ignore' });
-console.log('Successfully stopped test registry container'); // Output from command above is not good so we `ignore` it and emit our own
+  // Start test registry (Verdaccio)
+  const startRegistryProcessResult = childProcess.spawnSync(
+    'docker',
+    [
+      'run',
+      '--detach',
+      '--rm',
+      '--name',
+      TEST_REGISTRY_CONTAINER_NAME,
+      '-p',
+      '4873:4873',
+      '-v',
+      `${__dirname}/verdaccio-config:/verdaccio/conf`,
+      `verdaccio/verdaccio:${VERDACCIO_VERSION}`,
+    ],
+    { encoding: 'utf8', stdio: 'inherit' },
+  );
+
+  if (startRegistryProcessResult.status !== 0) {
+    process.exit(1);
+  }
+
+  // Build container image that is uploading our packages to fake registry with specific Node.js/npm version
+  const buildPublishImageProcessResult = childProcess.spawnSync(
+    'docker',
+    [
+      'build',
+      '--tag',
+      PUBLISH_PACKAGES_DOCKER_IMAGE_NAME,
+      '--file',
+      './Dockerfile.publish-packages',
+      ...(publishScriptNodeVersion ? ['--build-arg', `NODE_VERSION=${publishScriptNodeVersion}`] : []),
+      '.',
+    ],
+    {
+      encoding: 'utf8',
+      stdio: 'inherit',
+    },
+  );
+
+  if (buildPublishImageProcessResult.status !== 0) {
+    process.exit(1);
+  }
+
+  // Run container that uploads our packages to fake registry
+  const publishImageContainerRunProcess = childProcess.spawnSync(
+    'docker',
+    [
+      'run',
+      '--rm',
+      '-v',
+      `${repositoryRoot}:/sentry-javascript`,
+      '--network',
+      'host',
+      PUBLISH_PACKAGES_DOCKER_IMAGE_NAME,
+    ],
+    {
+      encoding: 'utf8',
+      stdio: 'inherit',
+    },
+  );
+
+  if (publishImageContainerRunProcess.status !== 0) {
+    process.exit(1);
+  }
+});
+
+const recipePaths = glob.sync(`${__dirname}/test-applications/*/test-recipe.json`, { absolute: true });
+
+let someTestFailed = false;
+
+const recipeResults = recipePaths.map(recipePath => {
+  type Recipe = {
+    testApplicationName: string;
+    buildCommand?: string;
+    tests: {
+      testName: string;
+      testCommand: string;
+      timeoutSeconds?: number;
+    }[];
+  };
+
+  const recipe: Recipe = JSON.parse(fs.readFileSync(recipePath, 'utf-8'));
+
+  if (recipe.buildCommand) {
+    console.log(`Running E2E test build command for test application "${recipe.testApplicationName}"`);
+    const buildCommandProcess = childProcess.spawnSync(recipe.buildCommand, {
+      cwd: path.dirname(recipePath),
+      encoding: 'utf8',
+      stdio: 'inherit',
+      shell: true, // needed so we can pass the build command in as whole without splitting it up into args
+    });
+
+    if (buildCommandProcess.status !== 0) {
+      process.exit(1);
+    }
+  }
+
+  type TestResult = {
+    testName: string;
+    result: 'PASS' | 'FAIL' | 'TIMEOUT';
+  };
+
+  const testResults: TestResult[] = recipe.tests.map(test => {
+    console.log(
+      `Running E2E test command for test application "${recipe.testApplicationName}", test "${test.testName}"`,
+    );
+
+    const testProcessResult = childProcess.spawnSync(test.testCommand, {
+      cwd: path.dirname(recipePath),
+      timeout: (test.timeoutSeconds ?? DEFAULT_TEST_TIMEOUT_SECONDS) * 1000,
+      encoding: 'utf8',
+      stdio: 'pipe',
+      shell: true, // needed so we can pass the test command in as whole without splitting it up into args
+    });
+
+    console.log(testProcessResult.stdout.replace(/^/gm, '[TEST OUTPUT] '));
+    console.log(testProcessResult.stderr.replace(/^/gm, '[TEST OUTPUT] '));
+
+    const error: undefined | (Error & { code?: string }) = testProcessResult.error;
+
+    if (error?.code === 'ETIMEDOUT') {
+      printCIErrorMessage(
+        `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname(
+          recipePath,
+        )}) timed out.`,
+      );
+      return {
+        testName: test.testName,
+        result: 'TIMEOUT',
+      };
+    } else if (testProcessResult.status !== 0) {
+      someTestFailed = true;
+      printCIErrorMessage(
+        `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname(
+          recipePath,
+        )}) failed.`,
+      );
+      return {
+        testName: test.testName,
+        result: 'FAIL',
+      };
+    } else {
+      someTestFailed = true;
+      console.log(
+        `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname(
+          recipePath,
+        )}) succeeded.`,
+      );
+      return {
+        testName: test.testName,
+        result: 'PASS',
+      };
+    }
+  });
+
+  return {
+    testApplicationName: recipe.testApplicationName,
+    testApplicationPath: recipePath,
+    testResults,
+  };
+});
+
+console.log('--------------------------------------');
+console.log('Test Result Summary:');
+
+recipeResults.forEach(recipeResult => {
+  console.log(`● ${recipeResult.testApplicationName} (${path.dirname(recipeResult.testApplicationPath)})`);
+  recipeResult.testResults.forEach(testResult => {
+    console.log(`  ● ${testResult.result.padEnd(7, ' ')} ${testResult.testName}`);
+  });
+});
+
+groupCIOutput('Cleanup', () => {
+  // Stop test registry
+  childProcess.spawnSync(`docker stop ${TEST_REGISTRY_CONTAINER_NAME}`, { encoding: 'utf8', stdio: 'ignore' });
+  console.log('Successfully stopped test registry container'); // Output from command above is not good so we `ignore` it and emit our own
+});
+
+if (someTestFailed) {
+  console.log('Not all tests succeeded.');
+  process.exit(1);
+} else {
+  console.log('All tests succeeded.');
+}

packages/e2e-tests/test-applications/temporary-app-1/.gitignore

@@ -0,0 +1 @@
+yarn.lock

packages/e2e-tests/test-applications/temporary-app-1/.npmrc

@@ -0,0 +1,3 @@
+@sentry:registry=http://localhost:4873
+@sentry-internal:registry=http://localhost:4873
+//localhost:4873/:_authToken=some-token

packages/e2e-tests/test-applications/temporary-app-1/bad.js

@@ -0,0 +1 @@
+throw new Error('Sad :(');

packages/e2e-tests/test-applications/temporary-app-1/good.js

@@ -0,0 +1 @@
+console.log('Happy :)');

packages/e2e-tests/test-applications/temporary-app-1/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "temporary-app-1",
+  "version": "1.0.0",
+  "private": true,
+  "scripts": {
+    "start:good": "node good.js",
+    "start:bad": "node bad.js",
+    "start:timeout": "node timeout.js"
+  },
+  "dependencies": {
+    "@sentry/node": "*"
+  }
+}

packages/e2e-tests/test-applications/temporary-app-1/test-recipe.json

@@ -0,0 +1,21 @@
+{
+  "$schema": "../../test-recipe-schema.json",
+  "testApplicationName": "Temporary Application 1",
+  "buildCommand": "yarn install",
+  "tests": [
+    {
+      "testName": "Example Test (Should Succeed)",
+      "testCommand": "yarn start:good",
+      "timeoutSeconds": 30
+    },
+    {
+      "testName": "Example Test (Should Fail)",
+      "testCommand": "yarn start:bad"
+    },
+    {
+      "testName": "Example Test (Should time out)",
+      "testCommand": "yarn start:timeout",
+      "timeoutSeconds": 5
+    }
+  ]
+}

packages/e2e-tests/test-applications/temporary-app-1/timeout.js

@@ -0,0 +1,3 @@
+setTimeout(() => {
+  console.log('Bored :/');
+}, 6000);

packages/e2e-tests/test-applications/temporary-app-2/.gitignore

@@ -0,0 +1 @@
+yarn.lock

packages/e2e-tests/test-applications/temporary-app-2/.npmrc

@@ -0,0 +1,3 @@
+@sentry:registry=http://localhost:4873
+@sentry-internal:registry=http://localhost:4873
+//localhost:4873/:_authToken=some-token

packages/e2e-tests/test-applications/temporary-app-2/bad.js

@@ -0,0 +1 @@
+throw new Error('Sad :(');

packages/e2e-tests/test-applications/temporary-app-2/good.js

@@ -0,0 +1 @@
+console.log('Happy :)');

packages/e2e-tests/test-applications/temporary-app-2/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "temporary-app-2",
+  "version": "1.0.0",
+  "private": true,
+  "scripts": {
+    "start:good": "node good.js",
+    "start:bad": "node bad.js",
+    "start:timeout": "node timeout.js"
+  },
+  "dependencies": {
+    "@sentry/node": "*"
+  }
+}

packages/e2e-tests/test-applications/temporary-app-2/test-recipe.json

@@ -0,0 +1,21 @@
+{
+  "$schema": "../../test-recipe-schema.json",
+  "testApplicationName": "Temporary Application 2",
+  "buildCommand": "yarn install",
+  "tests": [
+    {
+      "testName": "Example Test (Should Succeed)",
+      "testCommand": "yarn start:good",
+      "timeoutSeconds": 60
+    },
+    {
+      "testName": "Example Test (Should Fail)",
+      "testCommand": "yarn start:bad"
+    },
+    {
+      "testName": "Example Test (Should time out)",
+      "testCommand": "yarn start:timeout",
+      "timeoutSeconds": 5
+    }
+  ]
+}

packages/e2e-tests/test-applications/temporary-app-2/timeout.js

@@ -0,0 +1,3 @@
+setTimeout(() => {
+  console.log('Bored :/');
+}, 6000);