test(e2e): Add E2E Test recipe framework

packages/e2e-tests/.eslintrc.js

@@ -3,7 +3,7 @@ module.exports = {
     node: true,
   },
   extends: ['../../.eslintrc.js'],
-  ignorePatterns: [],
+  ignorePatterns: ['test-applications/**'],
   parserOptions: {
     sourceType: 'module',
   },

packages/e2e-tests/README.md

@@ -0,0 +1,99 @@
+# E2E Tests
+
+E2E tests enable us to verify the behavior of the packages in this repository as if they were to be published in their
+current state.
+
+## How to run
+
+Prerequisites: Docker
+
+```bash
+yarn test:e2e
+```
+
+## How they work
+
+Before running any tests we launch a fake test registry (in our case [Verdaccio](https://verdaccio.org/docs/e2e/)), we
+build our packages, pack them, and publish them to the fake registry. The fake registry is hosted in a Docker container,
+and the script to publish the packages is also run from within a container to ensure that the fake publishing happens
+with the same Node.js and npm versions as we're using in CI.
+
+After publishing our freshly built packages to the fake registry, the E2E test script will look for `test-recipe.json`
+files in test applications located in the `test-applications` folder. In this folder, we keep standalone test
+applications, that use our SDKs and can be used to verify their behavior. The `test-recipe.json` recipe files contain
+information on how to build the test applications and how to run tests on these applications.
+
+## How to set up a new test
+
+Test applications are completely standalone applications that can be used to verify our SDKs. To set one up, follow
+these commands:
+
+```sh
+cd packages/e2e-tests
+
+# Create a new test application folder
+mkdir test-applications/my-new-test-application # Name of the new folder doesn't technically matter but choose something meaningful
+
+# Create an npm configuration file that uses the fake test registry
+cat > test-applications/my-new-test-application/.npmrc << EOF
+@sentry:registry=http://localhost:4873
+@sentry-internal:registry=http://localhost:4873
+EOF
+
+# Add a gitignore that ignores lockfiles
+cat > test-applications/my-new-test-application/.gitignore << EOF
+yarn.lock
+package-lock.json
+EOF
+
+# Add a test recipe file to the test application
+touch test-applications/my-new-test-application/test-recipe.json
+```
+
+To get you started with the recipe, you can copy the following into `test-recipe.json`:
+
+```json
+{
+  "$schema": "../../test-recipe-schema.json",
+  "testApplicationName": "My New Test Application",
+  "buildCommand": "yarn install --no-lockfile",
+  "tests": [
+    {
+      "testName": "My new test",
+      "testCommand": "yarn test",
+      "timeoutSeconds": 60
+    }
+  ]
+}
+```
+
+The `test-recipe.json` files follow a schema (`e2e-tests/test-recipe-schema.json`). Here is a basic explanation of the
+fields:
+
+- The `buildCommand` command runs only once before any of the tests and is supposed to build the test application. If
+  this command returns a non-zero exit code, it counts as a failed test and the test application's tests are not run.
+- The `testCommand` command is supposed to run tests on the test application. If the configured command returns a
+  non-zero exit code, it counts as a failed test.
+- A test timeout can be configured via `timeoutSeconds`, it defaults to `60`.
+
+**An important thing to note:** In the context of the `buildCommand` the fake test registry is available at
+`http://localhost:4873`. It hosts all of our packages as if they were to be published with the state of the current
+branch. This means we can install the packages from this registry via the `.npmrc` configuration as seen above. If you
+add Sentry dependencies to your test application, you should set the dependency versions set to `*`:
+
+```jsonc
+// package.json
+{
+  "name": "my-new-test-application",
+  "version": "1.0.0",
+  "private": true,
+  "scripts": {
+    "test": "echo \"Hello world!\""
+  },
+  "dependencies": {
+    "@sentry/node": "*"
+  }
+}
+```
+
+All that is left for you to do now is to create a test app and run `yarn test:e2e`.

packages/e2e-tests/run.ts

@@ -1,5 +1,7 @@
 /* eslint-disable no-console */
 import * as childProcess from 'child_process';
+import * as fs from 'fs';
+import * as glob from 'glob';
 import * as path from 'path';
 
 const repositoryRoot = path.resolve(__dirname, '../..');
@@ -11,42 +13,245 @@ const PUBLISH_PACKAGES_DOCKER_IMAGE_NAME = 'publish-packages';
 
 const publishScriptNodeVersion = process.env.E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION;
 
-try {
+const DEFAULT_TEST_TIMEOUT_SECONDS = 60;
+
+// https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines
+function groupCIOutput(groupTitle: string, fn: () => void): void {
+  if (process.env.CI) {
+    console.log(`::group::${groupTitle}`);
+    fn();
+    console.log('::endgroup::');
+  } else {
+    fn();
+  }
+}
+
+// https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message
+function printCIErrorMessage(message: string): void {
+  if (process.env.CI) {
+    console.log(`::error::${message}`);
+  } else {
+    console.log(message);
+  }
+}
+
+groupCIOutput('Test Registry Setup', () => {
   // Stop test registry container (Verdaccio) if it was already running
-  childProcess.execSync(`docker stop ${TEST_REGISTRY_CONTAINER_NAME}`, { stdio: 'ignore' });
+  childProcess.spawnSync('docker', ['stop', TEST_REGISTRY_CONTAINER_NAME], { stdio: 'ignore' });
   console.log('Stopped previously running test registry');
-} catch (e) {
-  // Don't throw if container wasn't running
-}
 
-// Start test registry (Verdaccio)
-childProcess.execSync(
-  `docker run --detach --rm --name ${TEST_REGISTRY_CONTAINER_NAME} -p 4873:4873 -v ${__dirname}/verdaccio-config:/verdaccio/conf verdaccio/verdaccio:${VERDACCIO_VERSION}`,
-  { encoding: 'utf8', stdio: 'inherit' },
-);
-
-// Build container image that is uploading our packages to fake registry with specific Node.js/npm version
-childProcess.execSync(
-  `docker build --tag ${PUBLISH_PACKAGES_DOCKER_IMAGE_NAME} --file ./Dockerfile.publish-packages ${
-    publishScriptNodeVersion ? `--build-arg NODE_VERSION=${publishScriptNodeVersion}` : ''
-  } .`,
-  {
-    encoding: 'utf8',
-    stdio: 'inherit',
-  },
-);
-
-// Run container that uploads our packages to fake registry
-childProcess.execSync(
-  `docker run --rm -v ${repositoryRoot}:/sentry-javascript --network host ${PUBLISH_PACKAGES_DOCKER_IMAGE_NAME}`,
-  {
-    encoding: 'utf8',
-    stdio: 'inherit',
-  },
-);
-
-// TODO: Run e2e tests here
-
-// Stop test registry
-childProcess.execSync(`docker stop ${TEST_REGISTRY_CONTAINER_NAME}`, { encoding: 'utf8', stdio: 'ignore' });
-console.log('Successfully stopped test registry container'); // Output from command above is not good so we `ignore` it and emit our own
+  // Start test registry (Verdaccio)
+  const startRegistryProcessResult = childProcess.spawnSync(
+    'docker',
+    [
+      'run',
+      '--detach',
+      '--rm',
+      '--name',
+      TEST_REGISTRY_CONTAINER_NAME,
+      '-p',
+      '4873:4873',
+      '-v',
+      `${__dirname}/verdaccio-config:/verdaccio/conf`,
+      `verdaccio/verdaccio:${VERDACCIO_VERSION}`,
+    ],
+    { encoding: 'utf8', stdio: 'inherit' },
+  );
+
+  if (startRegistryProcessResult.status !== 0) {
+    process.exit(1);
+  }
+
+  // Build container image that is uploading our packages to fake registry with specific Node.js/npm version
+  const buildPublishImageProcessResult = childProcess.spawnSync(
+    'docker',
+    [
+      'build',
+      '--tag',
+      PUBLISH_PACKAGES_DOCKER_IMAGE_NAME,
+      '--file',
+      './Dockerfile.publish-packages',
+      ...(publishScriptNodeVersion ? ['--build-arg', `NODE_VERSION=${publishScriptNodeVersion}`] : []),
+      '.',
+    ],
+    {
+      encoding: 'utf8',
+      stdio: 'inherit',
+    },
+  );
+
+  if (buildPublishImageProcessResult.status !== 0) {
+    process.exit(1);
+  }
+
+  // Run container that uploads our packages to fake registry
+  const publishImageContainerRunProcess = childProcess.spawnSync(
+    'docker',
+    [
+      'run',
+      '--rm',
+      '-v',
+      `${repositoryRoot}:/sentry-javascript`,
+      '--network',
+      'host',
+      PUBLISH_PACKAGES_DOCKER_IMAGE_NAME,
+    ],
+    {
+      encoding: 'utf8',
+      stdio: 'inherit',
+    },
+  );
+
+  if (publishImageContainerRunProcess.status !== 0) {
+    process.exit(1);
+  }
+});
+
+const recipePaths = glob.sync(`${__dirname}/test-applications/*/test-recipe.json`, { absolute: true });
+
+let processShouldExitWithError = false;
+
+type TestResult = {
+  testName: string;
+  result: 'PASS' | 'FAIL' | 'TIMEOUT';
+};
+
+type RecipeResult = {
+  testApplicationName: string;
+  testApplicationPath: string;
+  buildFailed: boolean;
+  testResults: TestResult[];
+};
+
+const recipeResults: RecipeResult[] = recipePaths.map(recipePath => {
+  type Recipe = {
+    testApplicationName: string;
+    buildCommand?: string;
+    tests: {
+      testName: string;
+      testCommand: string;
+      timeoutSeconds?: number;
+    }[];
+  };
+
+  const recipe: Recipe = JSON.parse(fs.readFileSync(recipePath, 'utf-8'));
+
+  if (recipe.buildCommand) {
+    console.log(`Running E2E test build command for test application "${recipe.testApplicationName}"`);
+    const buildCommandProcess = childProcess.spawnSync(recipe.buildCommand, {
+      cwd: path.dirname(recipePath),
+      encoding: 'utf8',
+      shell: true, // needed so we can pass the build command in as whole without splitting it up into args
+    });
+
+    // Prepends some text to the output build command's output so we can distinguish it from logging in this script
+    console.log(buildCommandProcess.stdout.replace(/^/gm, '[BUILD OUTPUT] '));
+    console.log(buildCommandProcess.stderr.replace(/^/gm, '[BUILD OUTPUT] '));
+
+    if (buildCommandProcess.status !== 0) {
+      processShouldExitWithError = true;
+
+      printCIErrorMessage(
+        `Build command in test application "${recipe.testApplicationName}" (${path.dirname(recipePath)}) failed!`,
+      );
+
+      return {
+        testApplicationName: recipe.testApplicationName,
+        testApplicationPath: recipePath,
+        buildFailed: true,
+        testResults: [],
+      };
+    }
+  }
+
+  const testResults: TestResult[] = recipe.tests.map(test => {
+    console.log(
+      `Running E2E test command for test application "${recipe.testApplicationName}", test "${test.testName}"`,
+    );
+
+    const testProcessResult = childProcess.spawnSync(test.testCommand, {
+      cwd: path.dirname(recipePath),
+      timeout: (test.timeoutSeconds ?? DEFAULT_TEST_TIMEOUT_SECONDS) * 1000,
+      encoding: 'utf8',
+      shell: true, // needed so we can pass the test command in as whole without splitting it up into args
+    });
+
+    // Prepends some text to the output test command's output so we can distinguish it from logging in this script
+    console.log(testProcessResult.stdout.replace(/^/gm, '[TEST OUTPUT] '));
+    console.log(testProcessResult.stderr.replace(/^/gm, '[TEST OUTPUT] '));
+
+    const error: undefined | (Error & { code?: string }) = testProcessResult.error;
+
+    if (error?.code === 'ETIMEDOUT') {
+      processShouldExitWithError = true;
+      printCIErrorMessage(
+        `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname(
+          recipePath,
+        )}) timed out.`,
+      );
+      return {
+        testName: test.testName,
+        result: 'TIMEOUT',
+      };
+    } else if (testProcessResult.status !== 0) {
+      processShouldExitWithError = true;
+      printCIErrorMessage(
+        `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname(
+          recipePath,
+        )}) failed.`,
+      );
+      return {
+        testName: test.testName,
+        result: 'FAIL',
+      };
+    } else {
+      console.log(
+        `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname(
+          recipePath,
+        )}) succeeded.`,
+      );
+      return {
+        testName: test.testName,
+        result: 'PASS',
+      };
+    }
+  });
+
+  return {
+    testApplicationName: recipe.testApplicationName,
+    testApplicationPath: recipePath,
+    buildFailed: false,
+    testResults,
+  };
+});
+
+console.log('--------------------------------------');
+console.log('Test Result Summary:');
+
+recipeResults.forEach(recipeResult => {
+  if (recipeResult.buildFailed) {
+    console.log(
+      `● BUILD FAILED - ${recipeResult.testApplicationName} (${path.dirname(recipeResult.testApplicationPath)})`,
+    );
+  } else {
+    console.log(
+      `● BUILD SUCCEEDED - ${recipeResult.testApplicationName} (${path.dirname(recipeResult.testApplicationPath)})`,
+    );
+    recipeResult.testResults.forEach(testResult => {
+      console.log(`  ● ${testResult.result.padEnd(7, ' ')} ${testResult.testName}`);
+    });
+  }
+});
+
+groupCIOutput('Cleanup', () => {
+  // Stop test registry
+  childProcess.spawnSync(`docker stop ${TEST_REGISTRY_CONTAINER_NAME}`, { encoding: 'utf8', stdio: 'ignore' });
+  console.log('Successfully stopped test registry container'); // Output from command above is not good so we `ignore` it and emit our own
+});
+
+if (processShouldExitWithError) {
+  console.log('Not all tests succeeded.');
+  process.exit(1);
+} else {
+  console.log('All tests succeeded.');
+}