From 185790caf0b1a52b60cf02aea0321d0b5a931fe2 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Wed, 12 Oct 2022 10:52:00 +0000 Subject: [PATCH 01/13] test(e2e): Add framework for canary testing --- packages/e2e-tests/run.ts | 300 +++++++++++------- .../standard-frontend-react/test-recipe.json | 28 ++ 2 files changed, 209 insertions(+), 119 deletions(-) diff --git a/packages/e2e-tests/run.ts b/packages/e2e-tests/run.ts index 7a069907d65d..4a9aa5f72b7d 100644 --- a/packages/e2e-tests/run.ts +++ b/packages/e2e-tests/run.ts @@ -135,137 +135,189 @@ type TestResult = { result: 'PASS' | 'FAIL' | 'TIMEOUT'; }; +type VersionResult = { + dependencyOverrides?: Record; + buildFailed: boolean; + testResults: TestResult[]; +}; + type RecipeResult = { testApplicationName: string; testApplicationPath: string; - buildFailed: boolean; - testResults: TestResult[]; + versionResults: VersionResult[]; }; -const recipeResults: RecipeResult[] = recipePaths.map(recipePath => { - type Recipe = { - testApplicationName: string; - buildCommand?: string; - buildTimeoutSeconds?: number; - tests: { - testName: string; - testCommand: string; - timeoutSeconds?: number; - }[]; - }; +type Recipe = { + testApplicationName: string; + buildCommand?: string; + buildTimeoutSeconds?: number; + tests: { + testName: string; + testCommand: string; + timeoutSeconds?: number; + }[]; + versions?: { dependencyOverrides: Record }[]; + canaryVersions?: { dependencyOverrides: Record }[]; +}; +const recipeResults: RecipeResult[] = recipePaths.map(recipePath => { const recipe: Recipe = JSON.parse(fs.readFileSync(recipePath, 'utf-8')); + const recipeDirname = path.dirname(recipePath); - if (recipe.buildCommand) { - console.log(`Running E2E test build command for test application "${recipe.testApplicationName}"`); - const buildCommandProcess = childProcess.spawnSync(recipe.buildCommand, { - cwd: path.dirname(recipePath), - encoding: 'utf8', - shell: true, // needed so we can pass the build command in as whole without splitting it up into args - timeout: (recipe.buildTimeoutSeconds ?? DEFAULT_BUILD_TIMEOUT_SECONDS) * 1000, - env: { - ...process.env, - ...envVarsToInject, - }, - }); - - // Prepends some text to the output build command's output so we can distinguish it from logging in this script - console.log(buildCommandProcess.stdout.replace(/^/gm, '[BUILD OUTPUT] ')); - console.log(buildCommandProcess.stderr.replace(/^/gm, '[BUILD OUTPUT] ')); + function runRecipe(dependencyOverrides: Record | undefined): VersionResult { + const dependencyOverridesInformationString = dependencyOverrides + ? ` (Dependency overrides: ${JSON.stringify(dependencyOverrides)})` + : ''; - const error: undefined | (Error & { code?: string }) = buildCommandProcess.error; - - if (error?.code === 'ETIMEDOUT') { - processShouldExitWithError = true; - - printCIErrorMessage( - `Build command in test application "${recipe.testApplicationName}" (${path.dirname(recipePath)}) timed out!`, - ); - - return { - testApplicationName: recipe.testApplicationName, - testApplicationPath: recipePath, - buildFailed: true, - testResults: [], - }; - } else if (buildCommandProcess.status !== 0) { - processShouldExitWithError = true; - - printCIErrorMessage( - `Build command in test application "${recipe.testApplicationName}" (${path.dirname(recipePath)}) failed!`, + if (recipe.buildCommand) { + console.log( + `Running E2E test build command for test application "${recipe.testApplicationName}"${dependencyOverridesInformationString}`, ); - - return { - testApplicationName: recipe.testApplicationName, - testApplicationPath: recipePath, - buildFailed: true, - testResults: [], - }; + const buildCommandProcess = childProcess.spawnSync(recipe.buildCommand, { + cwd: path.dirname(recipePath), + encoding: 'utf8', + shell: true, // needed so we can pass the build command in as whole without splitting it up into args + timeout: (recipe.buildTimeoutSeconds ?? DEFAULT_BUILD_TIMEOUT_SECONDS) * 1000, + env: { + ...process.env, + ...envVarsToInject, + }, + }); + + // Prepends some text to the output build command's output so we can distinguish it from logging in this script + console.log(buildCommandProcess.stdout.replace(/^/gm, '[BUILD OUTPUT] ')); + console.log(buildCommandProcess.stderr.replace(/^/gm, '[BUILD OUTPUT] ')); + + const error: undefined | (Error & { code?: string }) = buildCommandProcess.error; + + if (error?.code === 'ETIMEDOUT') { + processShouldExitWithError = true; + + printCIErrorMessage( + `Build command in test application "${recipe.testApplicationName}" (${path.dirname(recipePath)}) timed out!`, + ); + + return { + dependencyOverrides, + buildFailed: true, + testResults: [], + }; + } else if (buildCommandProcess.status !== 0) { + processShouldExitWithError = true; + + printCIErrorMessage( + `Build command in test application "${recipe.testApplicationName}" (${path.dirname(recipePath)}) failed!`, + ); + + return { + dependencyOverrides, + buildFailed: true, + testResults: [], + }; + } } - } - const testResults: TestResult[] = recipe.tests.map(test => { - console.log( - `Running E2E test command for test application "${recipe.testApplicationName}", test "${test.testName}"`, - ); + const testResults: TestResult[] = recipe.tests.map(test => { + console.log( + `Running E2E test command for test application "${recipe.testApplicationName}", test "${test.testName}"${dependencyOverridesInformationString}`, + ); - const testProcessResult = childProcess.spawnSync(test.testCommand, { - cwd: path.dirname(recipePath), - timeout: (test.timeoutSeconds ?? DEFAULT_TEST_TIMEOUT_SECONDS) * 1000, - encoding: 'utf8', - shell: true, // needed so we can pass the test command in as whole without splitting it up into args - env: { - ...process.env, - ...envVarsToInject, - }, + const testProcessResult = childProcess.spawnSync(test.testCommand, { + cwd: path.dirname(recipePath), + timeout: (test.timeoutSeconds ?? DEFAULT_TEST_TIMEOUT_SECONDS) * 1000, + encoding: 'utf8', + shell: true, // needed so we can pass the test command in as whole without splitting it up into args + env: { + ...process.env, + ...envVarsToInject, + }, + }); + + // Prepends some text to the output test command's output so we can distinguish it from logging in this script + console.log(testProcessResult.stdout.replace(/^/gm, '[TEST OUTPUT] ')); + console.log(testProcessResult.stderr.replace(/^/gm, '[TEST OUTPUT] ')); + + const error: undefined | (Error & { code?: string }) = testProcessResult.error; + + if (error?.code === 'ETIMEDOUT') { + processShouldExitWithError = true; + printCIErrorMessage( + `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname( + recipePath, + )}) timed out.`, + ); + return { + testName: test.testName, + result: 'TIMEOUT', + }; + } else if (testProcessResult.status !== 0) { + processShouldExitWithError = true; + printCIErrorMessage( + `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname( + recipePath, + )}) failed.`, + ); + return { + testName: test.testName, + result: 'FAIL', + }; + } else { + console.log( + `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname( + recipePath, + )}) succeeded.`, + ); + return { + testName: test.testName, + result: 'PASS', + }; + } }); - // Prepends some text to the output test command's output so we can distinguish it from logging in this script - console.log(testProcessResult.stdout.replace(/^/gm, '[TEST OUTPUT] ')); - console.log(testProcessResult.stderr.replace(/^/gm, '[TEST OUTPUT] ')); + return { + dependencyOverrides, + buildFailed: false, + testResults, + }; + } - const error: undefined | (Error & { code?: string }) = testProcessResult.error; + const versionsToRun: { + dependencyOverrides?: Record; + }[] = (process.env.CANARY_E2E_TEST ? recipe.canaryVersions : recipe.versions) ?? [{}]; - if (error?.code === 'ETIMEDOUT') { - processShouldExitWithError = true; - printCIErrorMessage( - `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname( - recipePath, - )}) timed out.`, - ); - return { - testName: test.testName, - result: 'TIMEOUT', - }; - } else if (testProcessResult.status !== 0) { - processShouldExitWithError = true; - printCIErrorMessage( - `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname( - recipePath, - )}) failed.`, - ); - return { - testName: test.testName, - result: 'FAIL', - }; - } else { - console.log( - `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname( - recipePath, - )}) succeeded.`, + const versionResults = versionsToRun.map(({ dependencyOverrides }) => { + if (dependencyOverrides) { + // Back up original package.json + fs.copyFileSync(path.resolve(recipeDirname, 'package.json'), path.resolve(recipeDirname, 'package.json.bak')); + + // Override dependencies + const packageJson: { dependencies?: Record } = JSON.parse( + fs.readFileSync(path.resolve(recipeDirname, 'package.json'), { encoding: 'utf-8' }), ); - return { - testName: test.testName, - result: 'PASS', - }; + packageJson.dependencies = packageJson.dependencies + ? { ...packageJson.dependencies, ...dependencyOverrides } + : dependencyOverrides; + fs.writeFileSync(path.resolve(recipeDirname, 'package.json'), JSON.stringify(packageJson, null, 2), { + encoding: 'utf-8', + }); + } + + try { + return runRecipe(dependencyOverrides); + } finally { + if (dependencyOverrides) { + // Restore original package.json + fs.rmSync(path.resolve(recipeDirname, 'package.json'), { force: true }); + fs.copyFileSync(path.resolve(recipeDirname, 'package.json.bak'), path.resolve(recipeDirname, 'package.json')); + fs.rmSync(path.resolve(recipeDirname, 'package.json.bak'), { force: true }); + } } }); return { testApplicationName: recipe.testApplicationName, testApplicationPath: recipePath, - buildFailed: false, - testResults, + versionResults, }; }); @@ -273,18 +325,28 @@ console.log('--------------------------------------'); console.log('Test Result Summary:'); recipeResults.forEach(recipeResult => { - if (recipeResult.buildFailed) { - console.log( - `● BUILD FAILED - ${recipeResult.testApplicationName} (${path.dirname(recipeResult.testApplicationPath)})`, - ); - } else { - console.log( - `● BUILD SUCCEEDED - ${recipeResult.testApplicationName} (${path.dirname(recipeResult.testApplicationPath)})`, - ); - recipeResult.testResults.forEach(testResult => { - console.log(` ● ${testResult.result.padEnd(7, ' ')} ${testResult.testName}`); - }); - } + recipeResult.versionResults.forEach(versionResult => { + const dependencyOverridesInformationString = versionResult.dependencyOverrides + ? ` (Dependency overrides: ${JSON.stringify(versionResult.dependencyOverrides)})` + : ''; + + if (versionResult.buildFailed) { + console.log( + `● BUILD FAILED - ${recipeResult.testApplicationName} (${path.dirname( + recipeResult.testApplicationPath, + )})${dependencyOverridesInformationString}`, + ); + } else { + console.log( + `● BUILD SUCCEEDED - ${recipeResult.testApplicationName} (${path.dirname( + recipeResult.testApplicationPath, + )})${dependencyOverridesInformationString}`, + ); + versionResult.testResults.forEach(testResult => { + console.log(` ● ${testResult.result.padEnd(7, ' ')} ${testResult.testName}`); + }); + } + }); }); groupCIOutput('Cleanup', () => { diff --git a/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json b/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json index fc2b4c739b48..07ebe5d8b152 100644 --- a/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json +++ b/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json @@ -7,5 +7,33 @@ "testName": "Playwright tests", "testCommand": "yarn test" } + ], + "versions": [ + { + "dependencyOverrides": { + "react": "^18", + "react-dom": "^18" + } + }, + { + "dependencyOverrides": { + "react": "^17", + "react-dom": "^17" + } + }, + { + "dependencyOverrides": { + "react": "^16", + "react-dom": "^16" + } + } + ], + "canaryVersions": [ + { + "dependencyOverrides": { + "react": "latest", + "react-dom": "latest" + } + } ] } From 0ef64c6a19adb66d01dd1c4078a982a978570ead Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Wed, 12 Oct 2022 13:56:04 +0000 Subject: [PATCH 02/13] Remove version tests --- .../standard-frontend-react/test-recipe.json | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json b/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json index 07ebe5d8b152..95ff8d535877 100644 --- a/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json +++ b/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json @@ -8,26 +8,6 @@ "testCommand": "yarn test" } ], - "versions": [ - { - "dependencyOverrides": { - "react": "^18", - "react-dom": "^18" - } - }, - { - "dependencyOverrides": { - "react": "^17", - "react-dom": "^17" - } - }, - { - "dependencyOverrides": { - "react": "^16", - "react-dom": "^16" - } - } - ], "canaryVersions": [ { "dependencyOverrides": { From a5e71c2f36e6bb3ad688bbe1b952224db033d444 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Wed, 12 Oct 2022 14:27:28 +0000 Subject: [PATCH 03/13] Add canary workflow --- .github/workflows/canary.yml | 44 ++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/canary.yml diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml new file mode 100644 index 000000000000..25fe619a80bf --- /dev/null +++ b/.github/workflows/canary.yml @@ -0,0 +1,44 @@ +name: 'Build & Test' +on: + schedule: + # Runs "at minute 55 past every hour" (see https://crontab.guru) + - cron: '0 0 * * *' + pull_request: + workflow_dispatch: + inputs: + commit: + description: If the commit you want to test isn't the head of a branch, provide its SHA here + required: false + +env: + # We pin the exact version to enforce reproducable builds with node + npm. + DEFAULT_NODE_VERSION: '16.15.1' + HEAD_COMMIT: ${{ github.event.inputs.commit || github.sha }} + +jobs: + job_canary_test: + name: Run Canary Tests + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: 'Check out current commit' + uses: actions/checkout@v2 + with: + ref: ${{ env.HEAD_COMMIT }} + - name: Set up Node + uses: actions/setup-node@v1 + with: + node-version: ${{ env.DEFAULT_NODE_VERSION }} + - name: Install dependencies + run: yarn install --ignore-engines --frozen-lockfile + - name: Build packages + run: yarn build + - name: Run Canary Tests + env: + E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION: ${{ env.DEFAULT_NODE_VERSION }} + E2E_TEST_AUTH_TOKEN: ${{ secrets.E2E_TEST_AUTH_TOKEN }} + E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }} + CANARY_E2E_TEST: 'yes' + run: | + cd packages/e2e-tests + yarn test:e2e From b5e6d8c6a05e934e8afcda8e498bc9c9397062ec Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Wed, 12 Oct 2022 14:41:20 +0000 Subject: [PATCH 04/13] Update workflow --- .github/CANARY_FAILURE_TEMPLATE.md | 5 +++++ .github/workflows/canary.yml | 19 +++++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) create mode 100644 .github/CANARY_FAILURE_TEMPLATE.md diff --git a/.github/CANARY_FAILURE_TEMPLATE.md b/.github/CANARY_FAILURE_TEMPLATE.md new file mode 100644 index 000000000000..2a8eb3ae9ff0 --- /dev/null +++ b/.github/CANARY_FAILURE_TEMPLATE.md @@ -0,0 +1,5 @@ +--- +title: Canary tests failed +labels: 'Status: Untriaged' +--- +Canary tests failed: {{ env.RUN_LINK }} diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 25fe619a80bf..dd60203bc5cf 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -1,4 +1,4 @@ -name: 'Build & Test' +name: 'Canary Tests' on: schedule: # Runs "at minute 55 past every hour" (see https://crontab.guru) @@ -11,10 +11,12 @@ on: required: false env: - # We pin the exact version to enforce reproducable builds with node + npm. - DEFAULT_NODE_VERSION: '16.15.1' HEAD_COMMIT: ${{ github.event.inputs.commit || github.sha }} +permissions: + contents: read + issues: write + jobs: job_canary_test: name: Run Canary Tests @@ -28,7 +30,8 @@ jobs: - name: Set up Node uses: actions/setup-node@v1 with: - node-version: ${{ env.DEFAULT_NODE_VERSION }} + # ember won't build under node 16, at least not with the versions of the ember build tools we use + node-version: '14' - name: Install dependencies run: yarn install --ignore-engines --frozen-lockfile - name: Build packages @@ -42,3 +45,11 @@ jobs: run: | cd packages/e2e-tests yarn test:e2e + - name: Create Issue + if: failure() + - uses: JasonEtco/create-an-issue@1a16035489d05041b9af40b970f02e301c52ffba + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RUN_LINK: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + with: + filename: .github/CANARY_FAILURE_TEMPLATE.md From 10feb86c0ecf663a67ea3236a15a2ff4dc3a4eb2 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Wed, 12 Oct 2022 14:45:03 +0000 Subject: [PATCH 05/13] Update workflow --- .github/workflows/canary.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index dd60203bc5cf..49b308babdb7 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -47,7 +47,7 @@ jobs: yarn test:e2e - name: Create Issue if: failure() - - uses: JasonEtco/create-an-issue@1a16035489d05041b9af40b970f02e301c52ffba + uses: JasonEtco/create-an-issue@1a16035489d05041b9af40b970f02e301c52ffba env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} RUN_LINK: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} From d7db6f42fb48153633fbd162a2b757262ff673d6 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Wed, 12 Oct 2022 15:03:46 +0000 Subject: [PATCH 06/13] Fix logic for chosing what versions to test --- packages/e2e-tests/run.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/e2e-tests/run.ts b/packages/e2e-tests/run.ts index 4a9aa5f72b7d..455e27b5e5e1 100644 --- a/packages/e2e-tests/run.ts +++ b/packages/e2e-tests/run.ts @@ -283,7 +283,7 @@ const recipeResults: RecipeResult[] = recipePaths.map(recipePath => { const versionsToRun: { dependencyOverrides?: Record; - }[] = (process.env.CANARY_E2E_TEST ? recipe.canaryVersions : recipe.versions) ?? [{}]; + }[] = process.env.CANARY_E2E_TEST ? recipe.canaryVersions ?? [] : recipe.versions ?? [{}]; const versionResults = versionsToRun.map(({ dependencyOverrides }) => { if (dependencyOverrides) { From a766f9be7f7cca42501e461fdbee6571f9a24558 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Wed, 12 Oct 2022 15:04:01 +0000 Subject: [PATCH 07/13] Intentionally fail test --- .../standard-frontend-react/tests/behaviour-test.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/e2e-tests/test-applications/standard-frontend-react/tests/behaviour-test.spec.ts b/packages/e2e-tests/test-applications/standard-frontend-react/tests/behaviour-test.spec.ts index 53eef7287d9c..5a131b00ead6 100644 --- a/packages/e2e-tests/test-applications/standard-frontend-react/tests/behaviour-test.spec.ts +++ b/packages/e2e-tests/test-applications/standard-frontend-react/tests/behaviour-test.spec.ts @@ -95,7 +95,7 @@ test('Sends a pageload transaction to Sentry', async ({ page }) => { timeout: EVENT_POLLING_TIMEOUT, }, ) - .toBe(200); + .toBe(204); }), ); From 2e3e0f576cc3aea78e3885b8f9b17d69c21b7395 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Wed, 12 Oct 2022 15:11:00 +0000 Subject: [PATCH 08/13] Update workflow --- .github/workflows/canary.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 49b308babdb7..49dc6b92ea5f 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -28,10 +28,10 @@ jobs: with: ref: ${{ env.HEAD_COMMIT }} - name: Set up Node - uses: actions/setup-node@v1 + uses: actions/setup-node@v3 with: # ember won't build under node 16, at least not with the versions of the ember build tools we use - node-version: '14' + node-version: 14 - name: Install dependencies run: yarn install --ignore-engines --frozen-lockfile - name: Build packages From fa76acc601d2543c543cc7268c8227d209c27e69 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Wed, 12 Oct 2022 15:19:03 +0000 Subject: [PATCH 09/13] Fix intentionally failing test --- .../standard-frontend-react/tests/behaviour-test.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/e2e-tests/test-applications/standard-frontend-react/tests/behaviour-test.spec.ts b/packages/e2e-tests/test-applications/standard-frontend-react/tests/behaviour-test.spec.ts index 5a131b00ead6..53eef7287d9c 100644 --- a/packages/e2e-tests/test-applications/standard-frontend-react/tests/behaviour-test.spec.ts +++ b/packages/e2e-tests/test-applications/standard-frontend-react/tests/behaviour-test.spec.ts @@ -95,7 +95,7 @@ test('Sends a pageload transaction to Sentry', async ({ page }) => { timeout: EVENT_POLLING_TIMEOUT, }, ) - .toBe(204); + .toBe(200); }), ); From d7dfb311cf7c87065e3f21d3e8a7b1414a9e7a63 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Wed, 12 Oct 2022 15:24:16 +0000 Subject: [PATCH 10/13] Don't run workflow on PR --- .github/workflows/canary.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 49dc6b92ea5f..452b456a3c11 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -3,7 +3,6 @@ on: schedule: # Runs "at minute 55 past every hour" (see https://crontab.guru) - cron: '0 0 * * *' - pull_request: workflow_dispatch: inputs: commit: From a19d893ec4e394856f4dfddca81040f49b01c7f2 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Thu, 13 Oct 2022 12:55:00 +0000 Subject: [PATCH 11/13] Cleanup --- .github/workflows/canary.yml | 2 +- packages/e2e-tests/run.ts | 24 +++++++++++++----------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 452b456a3c11..e0d92547708e 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -1,7 +1,7 @@ name: 'Canary Tests' on: schedule: - # Runs "at minute 55 past every hour" (see https://crontab.guru) + # Run every day at midnight - cron: '0 0 * * *' workflow_dispatch: inputs: diff --git a/packages/e2e-tests/run.ts b/packages/e2e-tests/run.ts index 455e27b5e5e1..f9df41bcef9e 100644 --- a/packages/e2e-tests/run.ts +++ b/packages/e2e-tests/run.ts @@ -1,3 +1,4 @@ +/* eslint-disable max-lines */ /* eslint-disable no-console */ import * as childProcess from 'child_process'; import * as fs from 'fs'; @@ -286,18 +287,21 @@ const recipeResults: RecipeResult[] = recipePaths.map(recipePath => { }[] = process.env.CANARY_E2E_TEST ? recipe.canaryVersions ?? [] : recipe.versions ?? [{}]; const versionResults = versionsToRun.map(({ dependencyOverrides }) => { + const packageJsonPath = path.resolve(recipeDirname, 'package.json'); + const packageJsonBackupPath = path.resolve(recipeDirname, 'package.json.bak'); + if (dependencyOverrides) { // Back up original package.json - fs.copyFileSync(path.resolve(recipeDirname, 'package.json'), path.resolve(recipeDirname, 'package.json.bak')); + fs.copyFileSync(packageJsonPath, packageJsonBackupPath); // Override dependencies const packageJson: { dependencies?: Record } = JSON.parse( - fs.readFileSync(path.resolve(recipeDirname, 'package.json'), { encoding: 'utf-8' }), + fs.readFileSync(packageJsonPath, { encoding: 'utf-8' }), ); packageJson.dependencies = packageJson.dependencies ? { ...packageJson.dependencies, ...dependencyOverrides } : dependencyOverrides; - fs.writeFileSync(path.resolve(recipeDirname, 'package.json'), JSON.stringify(packageJson, null, 2), { + fs.writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, 2), { encoding: 'utf-8', }); } @@ -307,9 +311,9 @@ const recipeResults: RecipeResult[] = recipePaths.map(recipePath => { } finally { if (dependencyOverrides) { // Restore original package.json - fs.rmSync(path.resolve(recipeDirname, 'package.json'), { force: true }); - fs.copyFileSync(path.resolve(recipeDirname, 'package.json.bak'), path.resolve(recipeDirname, 'package.json')); - fs.rmSync(path.resolve(recipeDirname, 'package.json.bak'), { force: true }); + fs.rmSync(packageJsonPath, { force: true }); + fs.copyFileSync(packageJsonBackupPath, packageJsonPath); + fs.rmSync(packageJsonBackupPath, { force: true }); } } }); @@ -349,11 +353,9 @@ recipeResults.forEach(recipeResult => { }); }); -groupCIOutput('Cleanup', () => { - // Stop test registry - childProcess.spawnSync(`docker stop ${TEST_REGISTRY_CONTAINER_NAME}`, { encoding: 'utf8', stdio: 'ignore' }); - console.log('Successfully stopped test registry container'); // Output from command above is not good so we `ignore` it and emit our own -}); +// Stop test registry +childProcess.spawnSync(`docker stop ${TEST_REGISTRY_CONTAINER_NAME}`, { encoding: 'utf8', stdio: 'ignore' }); +console.log('Successfully stopped test registry container'); // Output from command above is not good so we `ignore` it and emit our own if (processShouldExitWithError) { console.log('Not all tests succeeded.'); From c82affb339995aa396f576c573571f23b2989b0f Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Thu, 13 Oct 2022 12:55:22 +0000 Subject: [PATCH 12/13] Add new properties to recipe schema --- packages/e2e-tests/test-recipe-schema.json | 28 ++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/packages/e2e-tests/test-recipe-schema.json b/packages/e2e-tests/test-recipe-schema.json index 8c0a93fe11c5..9f8bbf03deb5 100644 --- a/packages/e2e-tests/test-recipe-schema.json +++ b/packages/e2e-tests/test-recipe-schema.json @@ -36,6 +36,34 @@ }, "required": ["testName", "testCommand"] } + }, + "versions": { + "type": "array", + "description": "List of different dependency versions to test. If this property is provided, the tests will run for each item, if it is omitted, the tests will only run once with the dependencies provided in package.json.", + "items": { + "type": "object", + "properties": { + "dependencyOverrides": { + "type": "object", + "description": "Object to merge with the \"dependencies\" object in package.json.", + "additionalProperties": { "type": "string" } + } + } + } + }, + "canaryVersions": { + "type": "array", + "description": "List of different dependency versions to test as part of the canary test. Canary tests are only run if the CANARY_E2E_TEST environment variable is set. If If CANARY_E2E_TEST is set and this property is provided, the tests will run for each item, if it is omitted, no tests will be executed.", + "items": { + "type": "object", + "properties": { + "dependencyOverrides": { + "type": "object", + "description": "Object to merge with the \"dependencies\" object in package.json.", + "additionalProperties": { "type": "string" } + } + } + } } }, "required": ["testApplicationName", "tests"] From 3ba4f387dcae0ca34cf2991ecb6b56098a64beda Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Thu, 13 Oct 2022 13:41:20 +0000 Subject: [PATCH 13/13] Add tests label (let's hope this works) --- .github/CANARY_FAILURE_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CANARY_FAILURE_TEMPLATE.md b/.github/CANARY_FAILURE_TEMPLATE.md index 2a8eb3ae9ff0..49461065bd34 100644 --- a/.github/CANARY_FAILURE_TEMPLATE.md +++ b/.github/CANARY_FAILURE_TEMPLATE.md @@ -1,5 +1,5 @@ --- title: Canary tests failed -labels: 'Status: Untriaged' +labels: 'Status: Untriaged, Type: Tests' --- Canary tests failed: {{ env.RUN_LINK }}