diff --git a/packages/e2e-tests/.eslintrc.js b/packages/e2e-tests/.eslintrc.js index 5b0457483479..8c585e48f252 100644 --- a/packages/e2e-tests/.eslintrc.js +++ b/packages/e2e-tests/.eslintrc.js @@ -3,7 +3,7 @@ module.exports = { node: true, }, extends: ['../../.eslintrc.js'], - ignorePatterns: [], + ignorePatterns: ['test-applications/**'], parserOptions: { sourceType: 'module', }, diff --git a/packages/e2e-tests/README.md b/packages/e2e-tests/README.md new file mode 100644 index 000000000000..fbf819dc6310 --- /dev/null +++ b/packages/e2e-tests/README.md @@ -0,0 +1,99 @@ +# E2E Tests + +E2E tests enable us to verify the behavior of the packages in this repository as if they were to be published in their +current state. + +## How to run + +Prerequisites: Docker + +```bash +yarn test:e2e +``` + +## How they work + +Before running any tests we launch a fake test registry (in our case [Verdaccio](https://verdaccio.org/docs/e2e/)), we +build our packages, pack them, and publish them to the fake registry. The fake registry is hosted in a Docker container, +and the script to publish the packages is also run from within a container to ensure that the fake publishing happens +with the same Node.js and npm versions as we're using in CI. + +After publishing our freshly built packages to the fake registry, the E2E test script will look for `test-recipe.json` +files in test applications located in the `test-applications` folder. In this folder, we keep standalone test +applications, that use our SDKs and can be used to verify their behavior. The `test-recipe.json` recipe files contain +information on how to build the test applications and how to run tests on these applications. + +## How to set up a new test + +Test applications are completely standalone applications that can be used to verify our SDKs. To set one up, follow +these commands: + +```sh +cd packages/e2e-tests + +# Create a new test application folder +mkdir test-applications/my-new-test-application # Name of the new folder doesn't technically matter but choose something meaningful + +# Create an npm configuration file that uses the fake test registry +cat > test-applications/my-new-test-application/.npmrc << EOF +@sentry:registry=http://localhost:4873 +@sentry-internal:registry=http://localhost:4873 +EOF + +# Add a gitignore that ignores lockfiles +cat > test-applications/my-new-test-application/.gitignore << EOF +yarn.lock +package-lock.json +EOF + +# Add a test recipe file to the test application +touch test-applications/my-new-test-application/test-recipe.json +``` + +To get you started with the recipe, you can copy the following into `test-recipe.json`: + +```json +{ + "$schema": "../../test-recipe-schema.json", + "testApplicationName": "My New Test Application", + "buildCommand": "yarn install --no-lockfile", + "tests": [ + { + "testName": "My new test", + "testCommand": "yarn test", + "timeoutSeconds": 60 + } + ] +} +``` + +The `test-recipe.json` files follow a schema (`e2e-tests/test-recipe-schema.json`). Here is a basic explanation of the +fields: + +- The `buildCommand` command runs only once before any of the tests and is supposed to build the test application. If + this command returns a non-zero exit code, it counts as a failed test and the test application's tests are not run. +- The `testCommand` command is supposed to run tests on the test application. If the configured command returns a + non-zero exit code, it counts as a failed test. +- A test timeout can be configured via `timeoutSeconds`, it defaults to `60`. + +**An important thing to note:** In the context of the `buildCommand` the fake test registry is available at +`http://localhost:4873`. It hosts all of our packages as if they were to be published with the state of the current +branch. This means we can install the packages from this registry via the `.npmrc` configuration as seen above. If you +add Sentry dependencies to your test application, you should set the dependency versions set to `*`: + +```jsonc +// package.json +{ + "name": "my-new-test-application", + "version": "1.0.0", + "private": true, + "scripts": { + "test": "echo \"Hello world!\"" + }, + "dependencies": { + "@sentry/node": "*" + } +} +``` + +All that is left for you to do now is to create a test app and run `yarn test:e2e`. diff --git a/packages/e2e-tests/run.ts b/packages/e2e-tests/run.ts index 4a752e15a294..88319cce20c2 100644 --- a/packages/e2e-tests/run.ts +++ b/packages/e2e-tests/run.ts @@ -1,5 +1,7 @@ /* eslint-disable no-console */ import * as childProcess from 'child_process'; +import * as fs from 'fs'; +import * as glob from 'glob'; import * as path from 'path'; const repositoryRoot = path.resolve(__dirname, '../..'); @@ -11,42 +13,248 @@ const PUBLISH_PACKAGES_DOCKER_IMAGE_NAME = 'publish-packages'; const publishScriptNodeVersion = process.env.E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION; -try { +const DEFAULT_BUILD_TIMEOUT_SECONDS = 60; +const DEFAULT_TEST_TIMEOUT_SECONDS = 60; + +// https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines +function groupCIOutput(groupTitle: string, fn: () => void): void { + if (process.env.CI) { + console.log(`::group::${groupTitle}`); + fn(); + console.log('::endgroup::'); + } else { + fn(); + } +} + +// https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message +function printCIErrorMessage(message: string): void { + if (process.env.CI) { + console.log(`::error::${message}`); + } else { + console.log(message); + } +} + +groupCIOutput('Test Registry Setup', () => { // Stop test registry container (Verdaccio) if it was already running - childProcess.execSync(`docker stop ${TEST_REGISTRY_CONTAINER_NAME}`, { stdio: 'ignore' }); + childProcess.spawnSync('docker', ['stop', TEST_REGISTRY_CONTAINER_NAME], { stdio: 'ignore' }); console.log('Stopped previously running test registry'); -} catch (e) { - // Don't throw if container wasn't running -} -// Start test registry (Verdaccio) -childProcess.execSync( - `docker run --detach --rm --name ${TEST_REGISTRY_CONTAINER_NAME} -p 4873:4873 -v ${__dirname}/verdaccio-config:/verdaccio/conf verdaccio/verdaccio:${VERDACCIO_VERSION}`, - { encoding: 'utf8', stdio: 'inherit' }, -); - -// Build container image that is uploading our packages to fake registry with specific Node.js/npm version -childProcess.execSync( - `docker build --tag ${PUBLISH_PACKAGES_DOCKER_IMAGE_NAME} --file ./Dockerfile.publish-packages ${ - publishScriptNodeVersion ? `--build-arg NODE_VERSION=${publishScriptNodeVersion}` : '' - } .`, - { - encoding: 'utf8', - stdio: 'inherit', - }, -); - -// Run container that uploads our packages to fake registry -childProcess.execSync( - `docker run --rm -v ${repositoryRoot}:/sentry-javascript --network host ${PUBLISH_PACKAGES_DOCKER_IMAGE_NAME}`, - { - encoding: 'utf8', - stdio: 'inherit', - }, -); - -// TODO: Run e2e tests here - -// Stop test registry -childProcess.execSync(`docker stop ${TEST_REGISTRY_CONTAINER_NAME}`, { encoding: 'utf8', stdio: 'ignore' }); -console.log('Successfully stopped test registry container'); // Output from command above is not good so we `ignore` it and emit our own + // Start test registry (Verdaccio) + const startRegistryProcessResult = childProcess.spawnSync( + 'docker', + [ + 'run', + '--detach', + '--rm', + '--name', + TEST_REGISTRY_CONTAINER_NAME, + '-p', + '4873:4873', + '-v', + `${__dirname}/verdaccio-config:/verdaccio/conf`, + `verdaccio/verdaccio:${VERDACCIO_VERSION}`, + ], + { encoding: 'utf8', stdio: 'inherit' }, + ); + + if (startRegistryProcessResult.status !== 0) { + process.exit(1); + } + + // Build container image that is uploading our packages to fake registry with specific Node.js/npm version + const buildPublishImageProcessResult = childProcess.spawnSync( + 'docker', + [ + 'build', + '--tag', + PUBLISH_PACKAGES_DOCKER_IMAGE_NAME, + '--file', + './Dockerfile.publish-packages', + ...(publishScriptNodeVersion ? ['--build-arg', `NODE_VERSION=${publishScriptNodeVersion}`] : []), + '.', + ], + { + encoding: 'utf8', + stdio: 'inherit', + }, + ); + + if (buildPublishImageProcessResult.status !== 0) { + process.exit(1); + } + + // Run container that uploads our packages to fake registry + const publishImageContainerRunProcess = childProcess.spawnSync( + 'docker', + [ + 'run', + '--rm', + '-v', + `${repositoryRoot}:/sentry-javascript`, + '--network', + 'host', + PUBLISH_PACKAGES_DOCKER_IMAGE_NAME, + ], + { + encoding: 'utf8', + stdio: 'inherit', + }, + ); + + if (publishImageContainerRunProcess.status !== 0) { + process.exit(1); + } +}); + +const recipePaths = glob.sync(`${__dirname}/test-applications/*/test-recipe.json`, { absolute: true }); + +let processShouldExitWithError = false; + +type TestResult = { + testName: string; + result: 'PASS' | 'FAIL' | 'TIMEOUT'; +}; + +type RecipeResult = { + testApplicationName: string; + testApplicationPath: string; + buildFailed: boolean; + testResults: TestResult[]; +}; + +const recipeResults: RecipeResult[] = recipePaths.map(recipePath => { + type Recipe = { + testApplicationName: string; + buildCommand?: string; + buildTimeoutSeconds?: number; + tests: { + testName: string; + testCommand: string; + timeoutSeconds?: number; + }[]; + }; + + const recipe: Recipe = JSON.parse(fs.readFileSync(recipePath, 'utf-8')); + + if (recipe.buildCommand) { + console.log(`Running E2E test build command for test application "${recipe.testApplicationName}"`); + const buildCommandProcess = childProcess.spawnSync(recipe.buildCommand, { + cwd: path.dirname(recipePath), + encoding: 'utf8', + shell: true, // needed so we can pass the build command in as whole without splitting it up into args + timeout: (recipe.buildTimeoutSeconds ?? DEFAULT_BUILD_TIMEOUT_SECONDS) * 1000, + }); + + // Prepends some text to the output build command's output so we can distinguish it from logging in this script + console.log(buildCommandProcess.stdout.replace(/^/gm, '[BUILD OUTPUT] ')); + console.log(buildCommandProcess.stderr.replace(/^/gm, '[BUILD OUTPUT] ')); + + if (buildCommandProcess.status !== 0) { + processShouldExitWithError = true; + + printCIErrorMessage( + `Build command in test application "${recipe.testApplicationName}" (${path.dirname(recipePath)}) failed!`, + ); + + return { + testApplicationName: recipe.testApplicationName, + testApplicationPath: recipePath, + buildFailed: true, + testResults: [], + }; + } + } + + const testResults: TestResult[] = recipe.tests.map(test => { + console.log( + `Running E2E test command for test application "${recipe.testApplicationName}", test "${test.testName}"`, + ); + + const testProcessResult = childProcess.spawnSync(test.testCommand, { + cwd: path.dirname(recipePath), + timeout: (test.timeoutSeconds ?? DEFAULT_TEST_TIMEOUT_SECONDS) * 1000, + encoding: 'utf8', + shell: true, // needed so we can pass the test command in as whole without splitting it up into args + }); + + // Prepends some text to the output test command's output so we can distinguish it from logging in this script + console.log(testProcessResult.stdout.replace(/^/gm, '[TEST OUTPUT] ')); + console.log(testProcessResult.stderr.replace(/^/gm, '[TEST OUTPUT] ')); + + const error: undefined | (Error & { code?: string }) = testProcessResult.error; + + if (error?.code === 'ETIMEDOUT') { + processShouldExitWithError = true; + printCIErrorMessage( + `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname( + recipePath, + )}) timed out.`, + ); + return { + testName: test.testName, + result: 'TIMEOUT', + }; + } else if (testProcessResult.status !== 0) { + processShouldExitWithError = true; + printCIErrorMessage( + `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname( + recipePath, + )}) failed.`, + ); + return { + testName: test.testName, + result: 'FAIL', + }; + } else { + console.log( + `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname( + recipePath, + )}) succeeded.`, + ); + return { + testName: test.testName, + result: 'PASS', + }; + } + }); + + return { + testApplicationName: recipe.testApplicationName, + testApplicationPath: recipePath, + buildFailed: false, + testResults, + }; +}); + +console.log('--------------------------------------'); +console.log('Test Result Summary:'); + +recipeResults.forEach(recipeResult => { + if (recipeResult.buildFailed) { + console.log( + `● BUILD FAILED - ${recipeResult.testApplicationName} (${path.dirname(recipeResult.testApplicationPath)})`, + ); + } else { + console.log( + `● BUILD SUCCEEDED - ${recipeResult.testApplicationName} (${path.dirname(recipeResult.testApplicationPath)})`, + ); + recipeResult.testResults.forEach(testResult => { + console.log(` ● ${testResult.result.padEnd(7, ' ')} ${testResult.testName}`); + }); + } +}); + +groupCIOutput('Cleanup', () => { + // Stop test registry + childProcess.spawnSync(`docker stop ${TEST_REGISTRY_CONTAINER_NAME}`, { encoding: 'utf8', stdio: 'ignore' }); + console.log('Successfully stopped test registry container'); // Output from command above is not good so we `ignore` it and emit our own +}); + +if (processShouldExitWithError) { + console.log('Not all tests succeeded.'); + process.exit(1); +} else { + console.log('All tests succeeded.'); +} diff --git a/packages/e2e-tests/test-applications/temporary-app-1/.gitignore b/packages/e2e-tests/test-applications/temporary-app-1/.gitignore new file mode 100644 index 000000000000..8ee01d321b72 --- /dev/null +++ b/packages/e2e-tests/test-applications/temporary-app-1/.gitignore @@ -0,0 +1 @@ +yarn.lock diff --git a/packages/e2e-tests/test-applications/temporary-app-1/.npmrc b/packages/e2e-tests/test-applications/temporary-app-1/.npmrc new file mode 100644 index 000000000000..c6b3ef9b3eaa --- /dev/null +++ b/packages/e2e-tests/test-applications/temporary-app-1/.npmrc @@ -0,0 +1,2 @@ +@sentry:registry=http://localhost:4873 +@sentry-internal:registry=http://localhost:4873 diff --git a/packages/e2e-tests/test-applications/temporary-app-1/good.js b/packages/e2e-tests/test-applications/temporary-app-1/good.js new file mode 100644 index 000000000000..8da91c072fb8 --- /dev/null +++ b/packages/e2e-tests/test-applications/temporary-app-1/good.js @@ -0,0 +1 @@ +console.log('Happy :)'); diff --git a/packages/e2e-tests/test-applications/temporary-app-1/package.json b/packages/e2e-tests/test-applications/temporary-app-1/package.json new file mode 100644 index 000000000000..6adffeacc75f --- /dev/null +++ b/packages/e2e-tests/test-applications/temporary-app-1/package.json @@ -0,0 +1,11 @@ +{ + "name": "temporary-app-1", + "version": "1.0.0", + "private": true, + "scripts": { + "start:good": "node good.js" + }, + "dependencies": { + "@sentry/node": "*" + } +} diff --git a/packages/e2e-tests/test-applications/temporary-app-1/test-recipe.json b/packages/e2e-tests/test-applications/temporary-app-1/test-recipe.json new file mode 100644 index 000000000000..c00b1ff55a7e --- /dev/null +++ b/packages/e2e-tests/test-applications/temporary-app-1/test-recipe.json @@ -0,0 +1,11 @@ +{ + "$schema": "../../test-recipe-schema.json", + "testApplicationName": "Temporary Application 1", + "buildCommand": "yarn install --no-lockfile", + "tests": [ + { + "testName": "Example Test (Should Succeed)", + "testCommand": "yarn start:good" + } + ] +} diff --git a/packages/e2e-tests/test-applications/temporary-app-2/.gitignore b/packages/e2e-tests/test-applications/temporary-app-2/.gitignore new file mode 100644 index 000000000000..8ee01d321b72 --- /dev/null +++ b/packages/e2e-tests/test-applications/temporary-app-2/.gitignore @@ -0,0 +1 @@ +yarn.lock diff --git a/packages/e2e-tests/test-applications/temporary-app-2/.npmrc b/packages/e2e-tests/test-applications/temporary-app-2/.npmrc new file mode 100644 index 000000000000..c6b3ef9b3eaa --- /dev/null +++ b/packages/e2e-tests/test-applications/temporary-app-2/.npmrc @@ -0,0 +1,2 @@ +@sentry:registry=http://localhost:4873 +@sentry-internal:registry=http://localhost:4873 diff --git a/packages/e2e-tests/test-applications/temporary-app-2/good.js b/packages/e2e-tests/test-applications/temporary-app-2/good.js new file mode 100644 index 000000000000..8da91c072fb8 --- /dev/null +++ b/packages/e2e-tests/test-applications/temporary-app-2/good.js @@ -0,0 +1 @@ +console.log('Happy :)'); diff --git a/packages/e2e-tests/test-applications/temporary-app-2/package.json b/packages/e2e-tests/test-applications/temporary-app-2/package.json new file mode 100644 index 000000000000..a01456cb2f2e --- /dev/null +++ b/packages/e2e-tests/test-applications/temporary-app-2/package.json @@ -0,0 +1,11 @@ +{ + "name": "temporary-app-2", + "version": "1.0.0", + "private": true, + "scripts": { + "start:good": "node good.js" + }, + "dependencies": { + "@sentry/node": "*" + } +} diff --git a/packages/e2e-tests/test-applications/temporary-app-2/test-recipe.json b/packages/e2e-tests/test-applications/temporary-app-2/test-recipe.json new file mode 100644 index 000000000000..2ccdd52d62fb --- /dev/null +++ b/packages/e2e-tests/test-applications/temporary-app-2/test-recipe.json @@ -0,0 +1,11 @@ +{ + "$schema": "../../test-recipe-schema.json", + "testApplicationName": "Temporary Application 2", + "buildCommand": "yarn install --no-lockfile", + "tests": [ + { + "testName": "Example Test (Should Succeed)", + "testCommand": "yarn start:good" + } + ] +} diff --git a/packages/e2e-tests/test-recipe-schema.json b/packages/e2e-tests/test-recipe-schema.json new file mode 100644 index 000000000000..8c0a93fe11c5 --- /dev/null +++ b/packages/e2e-tests/test-recipe-schema.json @@ -0,0 +1,42 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Sentry JavaScript E2E Test Recipe", + "type": "object", + "properties": { + "testApplicationName": { + "type": "string", + "description": "Name displayed in test output" + }, + "buildCommand": { + "type": "string", + "description": "Command that is run to install dependencies and build the test application. This command is only run once before all tests. Working directory of the command is the root of the test application." + }, + "buildTimeoutSeconds": { + "type": "number", + "description": "Timeout for the build command in seconds. Default: 60" + }, + "tests": { + "type": "array", + "description": "Tests to run in this test application", + "items": { + "type": "object", + "properties": { + "testName": { + "type": "string", + "description": "Name displayed in test output" + }, + "testCommand": { + "type": "string", + "description": "Command that is run to start the test. Working directory of the command is the root of the test application. If this command returns a non-zero exit code the test counts as failed." + }, + "timeoutSeconds": { + "type": "number", + "description": "Test timeout in seconds. Default: 60" + } + }, + "required": ["testName", "testCommand"] + } + } + }, + "required": ["testApplicationName", "tests"] +}