diff --git a/packages/e2e-tests/.eslintrc.js b/packages/e2e-tests/.eslintrc.js index 5b0457483479..8c585e48f252 100644 --- a/packages/e2e-tests/.eslintrc.js +++ b/packages/e2e-tests/.eslintrc.js @@ -3,7 +3,7 @@ module.exports = { node: true, }, extends: ['../../.eslintrc.js'], - ignorePatterns: [], + ignorePatterns: ['test-applications/**'], parserOptions: { sourceType: 'module', }, diff --git a/packages/e2e-tests/run.ts b/packages/e2e-tests/run.ts index 4a752e15a294..e48e6135f701 100644 --- a/packages/e2e-tests/run.ts +++ b/packages/e2e-tests/run.ts @@ -1,5 +1,7 @@ /* eslint-disable no-console */ import * as childProcess from 'child_process'; +import * as fs from 'fs'; +import * as glob from 'glob'; import * as path from 'path'; const repositoryRoot = path.resolve(__dirname, '../..'); @@ -11,42 +13,215 @@ const PUBLISH_PACKAGES_DOCKER_IMAGE_NAME = 'publish-packages'; const publishScriptNodeVersion = process.env.E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION; -try { +const DEFAULT_TEST_TIMEOUT_SECONDS = 60; + +// https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines +function groupCIOutput(groupTitle: string, fn: () => void): void { + if (process.env.CI) { + console.log(`::group::${groupTitle}`); + fn(); + console.log('::endgroup::'); + } else { + fn(); + } +} + +// https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message +function printCIErrorMessage(message: string): void { + if (process.env.CI) { + console.log(`::error::${message}`); + } else { + console.log(message); + } +} + +groupCIOutput('Test Registry Setup', () => { // Stop test registry container (Verdaccio) if it was already running - childProcess.execSync(`docker stop ${TEST_REGISTRY_CONTAINER_NAME}`, { stdio: 'ignore' }); + childProcess.spawnSync('docker', ['stop', TEST_REGISTRY_CONTAINER_NAME], { stdio: 'ignore' }); console.log('Stopped previously running test registry'); -} catch (e) { - // Don't throw if container wasn't running -} -// Start test registry (Verdaccio) -childProcess.execSync( - `docker run --detach --rm --name ${TEST_REGISTRY_CONTAINER_NAME} -p 4873:4873 -v ${__dirname}/verdaccio-config:/verdaccio/conf verdaccio/verdaccio:${VERDACCIO_VERSION}`, - { encoding: 'utf8', stdio: 'inherit' }, -); - -// Build container image that is uploading our packages to fake registry with specific Node.js/npm version -childProcess.execSync( - `docker build --tag ${PUBLISH_PACKAGES_DOCKER_IMAGE_NAME} --file ./Dockerfile.publish-packages ${ - publishScriptNodeVersion ? `--build-arg NODE_VERSION=${publishScriptNodeVersion}` : '' - } .`, - { - encoding: 'utf8', - stdio: 'inherit', - }, -); - -// Run container that uploads our packages to fake registry -childProcess.execSync( - `docker run --rm -v ${repositoryRoot}:/sentry-javascript --network host ${PUBLISH_PACKAGES_DOCKER_IMAGE_NAME}`, - { - encoding: 'utf8', - stdio: 'inherit', - }, -); - -// TODO: Run e2e tests here - -// Stop test registry -childProcess.execSync(`docker stop ${TEST_REGISTRY_CONTAINER_NAME}`, { encoding: 'utf8', stdio: 'ignore' }); -console.log('Successfully stopped test registry container'); // Output from command above is not good so we `ignore` it and emit our own + // Start test registry (Verdaccio) + const startRegistryProcessResult = childProcess.spawnSync( + 'docker', + [ + 'run', + '--detach', + '--rm', + '--name', + TEST_REGISTRY_CONTAINER_NAME, + '-p', + '4873:4873', + '-v', + `${__dirname}/verdaccio-config:/verdaccio/conf`, + `verdaccio/verdaccio:${VERDACCIO_VERSION}`, + ], + { encoding: 'utf8', stdio: 'inherit' }, + ); + + if (startRegistryProcessResult.status !== 0) { + process.exit(1); + } + + // Build container image that is uploading our packages to fake registry with specific Node.js/npm version + const buildPublishImageProcessResult = childProcess.spawnSync( + 'docker', + [ + 'build', + '--tag', + PUBLISH_PACKAGES_DOCKER_IMAGE_NAME, + '--file', + './Dockerfile.publish-packages', + ...(publishScriptNodeVersion ? ['--build-arg', `NODE_VERSION=${publishScriptNodeVersion}`] : []), + '.', + ], + { + encoding: 'utf8', + stdio: 'inherit', + }, + ); + + if (buildPublishImageProcessResult.status !== 0) { + process.exit(1); + } + + // Run container that uploads our packages to fake registry + const publishImageContainerRunProcess = childProcess.spawnSync( + 'docker', + [ + 'run', + '--rm', + '-v', + `${repositoryRoot}:/sentry-javascript`, + '--network', + 'host', + PUBLISH_PACKAGES_DOCKER_IMAGE_NAME, + ], + { + encoding: 'utf8', + stdio: 'inherit', + }, + ); + + if (publishImageContainerRunProcess.status !== 0) { + process.exit(1); + } +}); + +const recipePaths = glob.sync(`${__dirname}/test-applications/*/test-recipe.json`, { absolute: true }); + +let someTestFailed = false; + +const recipeResults = recipePaths.map(recipePath => { + type Recipe = { + testApplicationName: string; + buildCommand?: string; + tests: { + testName: string; + testCommand: string; + timeoutSeconds?: number; + }[]; + }; + + const recipe: Recipe = JSON.parse(fs.readFileSync(recipePath, 'utf-8')); + + if (recipe.buildCommand) { + console.log(`Running E2E test build command for test application "${recipe.testApplicationName}"`); + const buildCommandProcess = childProcess.spawnSync(recipe.buildCommand, { + cwd: path.dirname(recipePath), + encoding: 'utf8', + stdio: 'inherit', + shell: true, // needed so we can pass the build command in as whole without splitting it up into args + }); + + if (buildCommandProcess.status !== 0) { + process.exit(1); + } + } + + type TestResult = { + testName: string; + result: 'PASS' | 'FAIL' | 'TIMEOUT'; + }; + + const testResults: TestResult[] = recipe.tests.map(test => { + console.log( + `Running E2E test command for test application "${recipe.testApplicationName}", test "${test.testName}"`, + ); + + const testProcessResult = childProcess.spawnSync(test.testCommand, { + cwd: path.dirname(recipePath), + timeout: (test.timeoutSeconds ?? DEFAULT_TEST_TIMEOUT_SECONDS) * 1000, + encoding: 'utf8', + stdio: 'pipe', + shell: true, // needed so we can pass the test command in as whole without splitting it up into args + }); + + console.log(testProcessResult.stdout.replace(/^/gm, '[TEST OUTPUT] ')); + console.log(testProcessResult.stderr.replace(/^/gm, '[TEST OUTPUT] ')); + + const error: undefined | (Error & { code?: string }) = testProcessResult.error; + + if (error?.code === 'ETIMEDOUT') { + printCIErrorMessage( + `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname( + recipePath, + )}) timed out.`, + ); + return { + testName: test.testName, + result: 'TIMEOUT', + }; + } else if (testProcessResult.status !== 0) { + someTestFailed = true; + printCIErrorMessage( + `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname( + recipePath, + )}) failed.`, + ); + return { + testName: test.testName, + result: 'FAIL', + }; + } else { + someTestFailed = true; + console.log( + `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname( + recipePath, + )}) succeeded.`, + ); + return { + testName: test.testName, + result: 'PASS', + }; + } + }); + + return { + testApplicationName: recipe.testApplicationName, + testApplicationPath: recipePath, + testResults, + }; +}); + +console.log('--------------------------------------'); +console.log('Test Result Summary:'); + +recipeResults.forEach(recipeResult => { + console.log(`● ${recipeResult.testApplicationName} (${path.dirname(recipeResult.testApplicationPath)})`); + recipeResult.testResults.forEach(testResult => { + console.log(` ● ${testResult.result.padEnd(7, ' ')} ${testResult.testName}`); + }); +}); + +groupCIOutput('Cleanup', () => { + // Stop test registry + childProcess.spawnSync(`docker stop ${TEST_REGISTRY_CONTAINER_NAME}`, { encoding: 'utf8', stdio: 'ignore' }); + console.log('Successfully stopped test registry container'); // Output from command above is not good so we `ignore` it and emit our own +}); + +if (someTestFailed) { + console.log('Not all tests succeeded.'); + process.exit(1); +} else { + console.log('All tests succeeded.'); +} diff --git a/packages/e2e-tests/test-applications/temporary-app-1/.gitignore b/packages/e2e-tests/test-applications/temporary-app-1/.gitignore new file mode 100644 index 000000000000..8ee01d321b72 --- /dev/null +++ b/packages/e2e-tests/test-applications/temporary-app-1/.gitignore @@ -0,0 +1 @@ +yarn.lock diff --git a/packages/e2e-tests/test-applications/temporary-app-1/.npmrc b/packages/e2e-tests/test-applications/temporary-app-1/.npmrc new file mode 100644 index 000000000000..c35d987cca9f --- /dev/null +++ b/packages/e2e-tests/test-applications/temporary-app-1/.npmrc @@ -0,0 +1,3 @@ +@sentry:registry=http://localhost:4873 +@sentry-internal:registry=http://localhost:4873 +//localhost:4873/:_authToken=some-token diff --git a/packages/e2e-tests/test-applications/temporary-app-1/bad.js b/packages/e2e-tests/test-applications/temporary-app-1/bad.js new file mode 100644 index 000000000000..bf99992592d6 --- /dev/null +++ b/packages/e2e-tests/test-applications/temporary-app-1/bad.js @@ -0,0 +1 @@ +throw new Error('Sad :('); diff --git a/packages/e2e-tests/test-applications/temporary-app-1/good.js b/packages/e2e-tests/test-applications/temporary-app-1/good.js new file mode 100644 index 000000000000..8da91c072fb8 --- /dev/null +++ b/packages/e2e-tests/test-applications/temporary-app-1/good.js @@ -0,0 +1 @@ +console.log('Happy :)'); diff --git a/packages/e2e-tests/test-applications/temporary-app-1/package.json b/packages/e2e-tests/test-applications/temporary-app-1/package.json new file mode 100644 index 000000000000..449dbd698c88 --- /dev/null +++ b/packages/e2e-tests/test-applications/temporary-app-1/package.json @@ -0,0 +1,13 @@ +{ + "name": "temporary-app-1", + "version": "1.0.0", + "private": true, + "scripts": { + "start:good": "node good.js", + "start:bad": "node bad.js", + "start:timeout": "node timeout.js" + }, + "dependencies": { + "@sentry/node": "*" + } +} diff --git a/packages/e2e-tests/test-applications/temporary-app-1/test-recipe.json b/packages/e2e-tests/test-applications/temporary-app-1/test-recipe.json new file mode 100644 index 000000000000..ddc1f2e72d9e --- /dev/null +++ b/packages/e2e-tests/test-applications/temporary-app-1/test-recipe.json @@ -0,0 +1,21 @@ +{ + "$schema": "../../test-recipe-schema.json", + "testApplicationName": "Temporary Application 1", + "buildCommand": "yarn install", + "tests": [ + { + "testName": "Example Test (Should Succeed)", + "testCommand": "yarn start:good", + "timeoutSeconds": 30 + }, + { + "testName": "Example Test (Should Fail)", + "testCommand": "yarn start:bad" + }, + { + "testName": "Example Test (Should time out)", + "testCommand": "yarn start:timeout", + "timeoutSeconds": 5 + } + ] +} diff --git a/packages/e2e-tests/test-applications/temporary-app-1/timeout.js b/packages/e2e-tests/test-applications/temporary-app-1/timeout.js new file mode 100644 index 000000000000..044ab5d7191a --- /dev/null +++ b/packages/e2e-tests/test-applications/temporary-app-1/timeout.js @@ -0,0 +1,3 @@ +setTimeout(() => { + console.log('Bored :/'); +}, 6000); diff --git a/packages/e2e-tests/test-applications/temporary-app-2/.gitignore b/packages/e2e-tests/test-applications/temporary-app-2/.gitignore new file mode 100644 index 000000000000..8ee01d321b72 --- /dev/null +++ b/packages/e2e-tests/test-applications/temporary-app-2/.gitignore @@ -0,0 +1 @@ +yarn.lock diff --git a/packages/e2e-tests/test-applications/temporary-app-2/.npmrc b/packages/e2e-tests/test-applications/temporary-app-2/.npmrc new file mode 100644 index 000000000000..c35d987cca9f --- /dev/null +++ b/packages/e2e-tests/test-applications/temporary-app-2/.npmrc @@ -0,0 +1,3 @@ +@sentry:registry=http://localhost:4873 +@sentry-internal:registry=http://localhost:4873 +//localhost:4873/:_authToken=some-token diff --git a/packages/e2e-tests/test-applications/temporary-app-2/bad.js b/packages/e2e-tests/test-applications/temporary-app-2/bad.js new file mode 100644 index 000000000000..bf99992592d6 --- /dev/null +++ b/packages/e2e-tests/test-applications/temporary-app-2/bad.js @@ -0,0 +1 @@ +throw new Error('Sad :('); diff --git a/packages/e2e-tests/test-applications/temporary-app-2/good.js b/packages/e2e-tests/test-applications/temporary-app-2/good.js new file mode 100644 index 000000000000..8da91c072fb8 --- /dev/null +++ b/packages/e2e-tests/test-applications/temporary-app-2/good.js @@ -0,0 +1 @@ +console.log('Happy :)'); diff --git a/packages/e2e-tests/test-applications/temporary-app-2/package.json b/packages/e2e-tests/test-applications/temporary-app-2/package.json new file mode 100644 index 000000000000..b9a5cb6f68e7 --- /dev/null +++ b/packages/e2e-tests/test-applications/temporary-app-2/package.json @@ -0,0 +1,13 @@ +{ + "name": "temporary-app-2", + "version": "1.0.0", + "private": true, + "scripts": { + "start:good": "node good.js", + "start:bad": "node bad.js", + "start:timeout": "node timeout.js" + }, + "dependencies": { + "@sentry/node": "*" + } +} diff --git a/packages/e2e-tests/test-applications/temporary-app-2/test-recipe.json b/packages/e2e-tests/test-applications/temporary-app-2/test-recipe.json new file mode 100644 index 000000000000..9d93f49cc667 --- /dev/null +++ b/packages/e2e-tests/test-applications/temporary-app-2/test-recipe.json @@ -0,0 +1,21 @@ +{ + "$schema": "../../test-recipe-schema.json", + "testApplicationName": "Temporary Application 2", + "buildCommand": "yarn install", + "tests": [ + { + "testName": "Example Test (Should Succeed)", + "testCommand": "yarn start:good", + "timeoutSeconds": 60 + }, + { + "testName": "Example Test (Should Fail)", + "testCommand": "yarn start:bad" + }, + { + "testName": "Example Test (Should time out)", + "testCommand": "yarn start:timeout", + "timeoutSeconds": 5 + } + ] +} diff --git a/packages/e2e-tests/test-applications/temporary-app-2/timeout.js b/packages/e2e-tests/test-applications/temporary-app-2/timeout.js new file mode 100644 index 000000000000..044ab5d7191a --- /dev/null +++ b/packages/e2e-tests/test-applications/temporary-app-2/timeout.js @@ -0,0 +1,3 @@ +setTimeout(() => { + console.log('Bored :/'); +}, 6000); diff --git a/packages/e2e-tests/test-recipe-schema.json b/packages/e2e-tests/test-recipe-schema.json new file mode 100644 index 000000000000..f2fd9a25659b --- /dev/null +++ b/packages/e2e-tests/test-recipe-schema.json @@ -0,0 +1,38 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Sentry JavaScript E2E Test Recipe", + "type": "object", + "properties": { + "testApplicationName": { + "type": "string", + "description": "Name displayed in test output" + }, + "buildCommand": { + "type": "string", + "description": "Command that is run to install dependencies and build the test application. This command is only run once before all tests. Working directory of the command is the root of the test application." + }, + "tests": { + "type": "array", + "description": "Tests to run in this test application", + "items": { + "type": "object", + "properties": { + "testName": { + "type": "string", + "description": "Name displayed in test output" + }, + "testCommand": { + "type": "string", + "description": "Command that is run to start the test. Working directory of the command is the root of the test application. If this command returns a non-zero exit code the test counts as failed." + }, + "timeoutSeconds": { + "type": "number", + "description": "Test timeout in seconds. Default: 60" + } + }, + "required": ["testName", "testCommand"] + } + } + }, + "required": ["testApplicationName", "tests"] +}