diff --git a/.github/CANARY_FAILURE_TEMPLATE.md b/.github/CANARY_FAILURE_TEMPLATE.md new file mode 100644 index 000000000000..49461065bd34 --- /dev/null +++ b/.github/CANARY_FAILURE_TEMPLATE.md @@ -0,0 +1,5 @@ +--- +title: Canary tests failed +labels: 'Status: Untriaged, Type: Tests' +--- +Canary tests failed: {{ env.RUN_LINK }} diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml new file mode 100644 index 000000000000..e0d92547708e --- /dev/null +++ b/.github/workflows/canary.yml @@ -0,0 +1,54 @@ +name: 'Canary Tests' +on: + schedule: + # Run every day at midnight + - cron: '0 0 * * *' + workflow_dispatch: + inputs: + commit: + description: If the commit you want to test isn't the head of a branch, provide its SHA here + required: false + +env: + HEAD_COMMIT: ${{ github.event.inputs.commit || github.sha }} + +permissions: + contents: read + issues: write + +jobs: + job_canary_test: + name: Run Canary Tests + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: 'Check out current commit' + uses: actions/checkout@v2 + with: + ref: ${{ env.HEAD_COMMIT }} + - name: Set up Node + uses: actions/setup-node@v3 + with: + # ember won't build under node 16, at least not with the versions of the ember build tools we use + node-version: 14 + - name: Install dependencies + run: yarn install --ignore-engines --frozen-lockfile + - name: Build packages + run: yarn build + - name: Run Canary Tests + env: + E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION: ${{ env.DEFAULT_NODE_VERSION }} + E2E_TEST_AUTH_TOKEN: ${{ secrets.E2E_TEST_AUTH_TOKEN }} + E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }} + CANARY_E2E_TEST: 'yes' + run: | + cd packages/e2e-tests + yarn test:e2e + - name: Create Issue + if: failure() + uses: JasonEtco/create-an-issue@1a16035489d05041b9af40b970f02e301c52ffba + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RUN_LINK: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + with: + filename: .github/CANARY_FAILURE_TEMPLATE.md diff --git a/packages/e2e-tests/run.ts b/packages/e2e-tests/run.ts index 7a069907d65d..f9df41bcef9e 100644 --- a/packages/e2e-tests/run.ts +++ b/packages/e2e-tests/run.ts @@ -1,3 +1,4 @@ +/* eslint-disable max-lines */ /* eslint-disable no-console */ import * as childProcess from 'child_process'; import * as fs from 'fs'; @@ -135,137 +136,192 @@ type TestResult = { result: 'PASS' | 'FAIL' | 'TIMEOUT'; }; +type VersionResult = { + dependencyOverrides?: Record; + buildFailed: boolean; + testResults: TestResult[]; +}; + type RecipeResult = { testApplicationName: string; testApplicationPath: string; - buildFailed: boolean; - testResults: TestResult[]; + versionResults: VersionResult[]; }; -const recipeResults: RecipeResult[] = recipePaths.map(recipePath => { - type Recipe = { - testApplicationName: string; - buildCommand?: string; - buildTimeoutSeconds?: number; - tests: { - testName: string; - testCommand: string; - timeoutSeconds?: number; - }[]; - }; +type Recipe = { + testApplicationName: string; + buildCommand?: string; + buildTimeoutSeconds?: number; + tests: { + testName: string; + testCommand: string; + timeoutSeconds?: number; + }[]; + versions?: { dependencyOverrides: Record }[]; + canaryVersions?: { dependencyOverrides: Record }[]; +}; +const recipeResults: RecipeResult[] = recipePaths.map(recipePath => { const recipe: Recipe = JSON.parse(fs.readFileSync(recipePath, 'utf-8')); + const recipeDirname = path.dirname(recipePath); - if (recipe.buildCommand) { - console.log(`Running E2E test build command for test application "${recipe.testApplicationName}"`); - const buildCommandProcess = childProcess.spawnSync(recipe.buildCommand, { - cwd: path.dirname(recipePath), - encoding: 'utf8', - shell: true, // needed so we can pass the build command in as whole without splitting it up into args - timeout: (recipe.buildTimeoutSeconds ?? DEFAULT_BUILD_TIMEOUT_SECONDS) * 1000, - env: { - ...process.env, - ...envVarsToInject, - }, - }); - - // Prepends some text to the output build command's output so we can distinguish it from logging in this script - console.log(buildCommandProcess.stdout.replace(/^/gm, '[BUILD OUTPUT] ')); - console.log(buildCommandProcess.stderr.replace(/^/gm, '[BUILD OUTPUT] ')); + function runRecipe(dependencyOverrides: Record | undefined): VersionResult { + const dependencyOverridesInformationString = dependencyOverrides + ? ` (Dependency overrides: ${JSON.stringify(dependencyOverrides)})` + : ''; - const error: undefined | (Error & { code?: string }) = buildCommandProcess.error; - - if (error?.code === 'ETIMEDOUT') { - processShouldExitWithError = true; - - printCIErrorMessage( - `Build command in test application "${recipe.testApplicationName}" (${path.dirname(recipePath)}) timed out!`, + if (recipe.buildCommand) { + console.log( + `Running E2E test build command for test application "${recipe.testApplicationName}"${dependencyOverridesInformationString}`, ); + const buildCommandProcess = childProcess.spawnSync(recipe.buildCommand, { + cwd: path.dirname(recipePath), + encoding: 'utf8', + shell: true, // needed so we can pass the build command in as whole without splitting it up into args + timeout: (recipe.buildTimeoutSeconds ?? DEFAULT_BUILD_TIMEOUT_SECONDS) * 1000, + env: { + ...process.env, + ...envVarsToInject, + }, + }); + + // Prepends some text to the output build command's output so we can distinguish it from logging in this script + console.log(buildCommandProcess.stdout.replace(/^/gm, '[BUILD OUTPUT] ')); + console.log(buildCommandProcess.stderr.replace(/^/gm, '[BUILD OUTPUT] ')); + + const error: undefined | (Error & { code?: string }) = buildCommandProcess.error; + + if (error?.code === 'ETIMEDOUT') { + processShouldExitWithError = true; + + printCIErrorMessage( + `Build command in test application "${recipe.testApplicationName}" (${path.dirname(recipePath)}) timed out!`, + ); + + return { + dependencyOverrides, + buildFailed: true, + testResults: [], + }; + } else if (buildCommandProcess.status !== 0) { + processShouldExitWithError = true; + + printCIErrorMessage( + `Build command in test application "${recipe.testApplicationName}" (${path.dirname(recipePath)}) failed!`, + ); + + return { + dependencyOverrides, + buildFailed: true, + testResults: [], + }; + } + } - return { - testApplicationName: recipe.testApplicationName, - testApplicationPath: recipePath, - buildFailed: true, - testResults: [], - }; - } else if (buildCommandProcess.status !== 0) { - processShouldExitWithError = true; - - printCIErrorMessage( - `Build command in test application "${recipe.testApplicationName}" (${path.dirname(recipePath)}) failed!`, + const testResults: TestResult[] = recipe.tests.map(test => { + console.log( + `Running E2E test command for test application "${recipe.testApplicationName}", test "${test.testName}"${dependencyOverridesInformationString}`, ); - return { - testApplicationName: recipe.testApplicationName, - testApplicationPath: recipePath, - buildFailed: true, - testResults: [], - }; - } - } + const testProcessResult = childProcess.spawnSync(test.testCommand, { + cwd: path.dirname(recipePath), + timeout: (test.timeoutSeconds ?? DEFAULT_TEST_TIMEOUT_SECONDS) * 1000, + encoding: 'utf8', + shell: true, // needed so we can pass the test command in as whole without splitting it up into args + env: { + ...process.env, + ...envVarsToInject, + }, + }); + + // Prepends some text to the output test command's output so we can distinguish it from logging in this script + console.log(testProcessResult.stdout.replace(/^/gm, '[TEST OUTPUT] ')); + console.log(testProcessResult.stderr.replace(/^/gm, '[TEST OUTPUT] ')); + + const error: undefined | (Error & { code?: string }) = testProcessResult.error; + + if (error?.code === 'ETIMEDOUT') { + processShouldExitWithError = true; + printCIErrorMessage( + `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname( + recipePath, + )}) timed out.`, + ); + return { + testName: test.testName, + result: 'TIMEOUT', + }; + } else if (testProcessResult.status !== 0) { + processShouldExitWithError = true; + printCIErrorMessage( + `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname( + recipePath, + )}) failed.`, + ); + return { + testName: test.testName, + result: 'FAIL', + }; + } else { + console.log( + `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname( + recipePath, + )}) succeeded.`, + ); + return { + testName: test.testName, + result: 'PASS', + }; + } + }); - const testResults: TestResult[] = recipe.tests.map(test => { - console.log( - `Running E2E test command for test application "${recipe.testApplicationName}", test "${test.testName}"`, - ); + return { + dependencyOverrides, + buildFailed: false, + testResults, + }; + } - const testProcessResult = childProcess.spawnSync(test.testCommand, { - cwd: path.dirname(recipePath), - timeout: (test.timeoutSeconds ?? DEFAULT_TEST_TIMEOUT_SECONDS) * 1000, - encoding: 'utf8', - shell: true, // needed so we can pass the test command in as whole without splitting it up into args - env: { - ...process.env, - ...envVarsToInject, - }, - }); + const versionsToRun: { + dependencyOverrides?: Record; + }[] = process.env.CANARY_E2E_TEST ? recipe.canaryVersions ?? [] : recipe.versions ?? [{}]; - // Prepends some text to the output test command's output so we can distinguish it from logging in this script - console.log(testProcessResult.stdout.replace(/^/gm, '[TEST OUTPUT] ')); - console.log(testProcessResult.stderr.replace(/^/gm, '[TEST OUTPUT] ')); + const versionResults = versionsToRun.map(({ dependencyOverrides }) => { + const packageJsonPath = path.resolve(recipeDirname, 'package.json'); + const packageJsonBackupPath = path.resolve(recipeDirname, 'package.json.bak'); - const error: undefined | (Error & { code?: string }) = testProcessResult.error; + if (dependencyOverrides) { + // Back up original package.json + fs.copyFileSync(packageJsonPath, packageJsonBackupPath); - if (error?.code === 'ETIMEDOUT') { - processShouldExitWithError = true; - printCIErrorMessage( - `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname( - recipePath, - )}) timed out.`, + // Override dependencies + const packageJson: { dependencies?: Record } = JSON.parse( + fs.readFileSync(packageJsonPath, { encoding: 'utf-8' }), ); - return { - testName: test.testName, - result: 'TIMEOUT', - }; - } else if (testProcessResult.status !== 0) { - processShouldExitWithError = true; - printCIErrorMessage( - `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname( - recipePath, - )}) failed.`, - ); - return { - testName: test.testName, - result: 'FAIL', - }; - } else { - console.log( - `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname( - recipePath, - )}) succeeded.`, - ); - return { - testName: test.testName, - result: 'PASS', - }; + packageJson.dependencies = packageJson.dependencies + ? { ...packageJson.dependencies, ...dependencyOverrides } + : dependencyOverrides; + fs.writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, 2), { + encoding: 'utf-8', + }); + } + + try { + return runRecipe(dependencyOverrides); + } finally { + if (dependencyOverrides) { + // Restore original package.json + fs.rmSync(packageJsonPath, { force: true }); + fs.copyFileSync(packageJsonBackupPath, packageJsonPath); + fs.rmSync(packageJsonBackupPath, { force: true }); + } } }); return { testApplicationName: recipe.testApplicationName, testApplicationPath: recipePath, - buildFailed: false, - testResults, + versionResults, }; }); @@ -273,26 +329,34 @@ console.log('--------------------------------------'); console.log('Test Result Summary:'); recipeResults.forEach(recipeResult => { - if (recipeResult.buildFailed) { - console.log( - `● BUILD FAILED - ${recipeResult.testApplicationName} (${path.dirname(recipeResult.testApplicationPath)})`, - ); - } else { - console.log( - `● BUILD SUCCEEDED - ${recipeResult.testApplicationName} (${path.dirname(recipeResult.testApplicationPath)})`, - ); - recipeResult.testResults.forEach(testResult => { - console.log(` ● ${testResult.result.padEnd(7, ' ')} ${testResult.testName}`); - }); - } -}); + recipeResult.versionResults.forEach(versionResult => { + const dependencyOverridesInformationString = versionResult.dependencyOverrides + ? ` (Dependency overrides: ${JSON.stringify(versionResult.dependencyOverrides)})` + : ''; -groupCIOutput('Cleanup', () => { - // Stop test registry - childProcess.spawnSync(`docker stop ${TEST_REGISTRY_CONTAINER_NAME}`, { encoding: 'utf8', stdio: 'ignore' }); - console.log('Successfully stopped test registry container'); // Output from command above is not good so we `ignore` it and emit our own + if (versionResult.buildFailed) { + console.log( + `● BUILD FAILED - ${recipeResult.testApplicationName} (${path.dirname( + recipeResult.testApplicationPath, + )})${dependencyOverridesInformationString}`, + ); + } else { + console.log( + `● BUILD SUCCEEDED - ${recipeResult.testApplicationName} (${path.dirname( + recipeResult.testApplicationPath, + )})${dependencyOverridesInformationString}`, + ); + versionResult.testResults.forEach(testResult => { + console.log(` ● ${testResult.result.padEnd(7, ' ')} ${testResult.testName}`); + }); + } + }); }); +// Stop test registry +childProcess.spawnSync(`docker stop ${TEST_REGISTRY_CONTAINER_NAME}`, { encoding: 'utf8', stdio: 'ignore' }); +console.log('Successfully stopped test registry container'); // Output from command above is not good so we `ignore` it and emit our own + if (processShouldExitWithError) { console.log('Not all tests succeeded.'); process.exit(1); diff --git a/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json b/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json index fc2b4c739b48..95ff8d535877 100644 --- a/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json +++ b/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json @@ -7,5 +7,13 @@ "testName": "Playwright tests", "testCommand": "yarn test" } + ], + "canaryVersions": [ + { + "dependencyOverrides": { + "react": "latest", + "react-dom": "latest" + } + } ] } diff --git a/packages/e2e-tests/test-recipe-schema.json b/packages/e2e-tests/test-recipe-schema.json index 8c0a93fe11c5..9f8bbf03deb5 100644 --- a/packages/e2e-tests/test-recipe-schema.json +++ b/packages/e2e-tests/test-recipe-schema.json @@ -36,6 +36,34 @@ }, "required": ["testName", "testCommand"] } + }, + "versions": { + "type": "array", + "description": "List of different dependency versions to test. If this property is provided, the tests will run for each item, if it is omitted, the tests will only run once with the dependencies provided in package.json.", + "items": { + "type": "object", + "properties": { + "dependencyOverrides": { + "type": "object", + "description": "Object to merge with the \"dependencies\" object in package.json.", + "additionalProperties": { "type": "string" } + } + } + } + }, + "canaryVersions": { + "type": "array", + "description": "List of different dependency versions to test as part of the canary test. Canary tests are only run if the CANARY_E2E_TEST environment variable is set. If If CANARY_E2E_TEST is set and this property is provided, the tests will run for each item, if it is omitted, no tests will be executed.", + "items": { + "type": "object", + "properties": { + "dependencyOverrides": { + "type": "object", + "description": "Object to merge with the \"dependencies\" object in package.json.", + "additionalProperties": { "type": "string" } + } + } + } } }, "required": ["testApplicationName", "tests"]