diff --git a/.github/workflows/flaky-test-detector.yml b/.github/workflows/flaky-test-detector.yml new file mode 100644 index 000000000000..bb414b553abd --- /dev/null +++ b/.github/workflows/flaky-test-detector.yml @@ -0,0 +1,81 @@ +name: 'Detect flaky tests' +on: + workflow_dispatch: + pull_request: + paths: + - 'packages/browser-integration-tests/suites/**' + +env: + HEAD_COMMIT: ${{ github.event.inputs.commit || github.sha }} + + NX_CACHE_RESTORE_KEYS: | + nx-Linux-${{ github.ref }}-${{ github.event.inputs.commit || github.sha }} + nx-Linux-${{ github.ref }} + nx-Linux + +# Cancel in progress workflows on pull_requests. +# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + flaky-detector: + runs-on: ubuntu-20.04 + timeout-minutes: 60 + name: 'Check tests for flakiness' + steps: + - name: Check out current branch + uses: actions/checkout@v3 + - name: Set up Node + uses: volta-cli/action@v4 + + - name: Install dependencies + run: yarn install --ignore-engines --frozen-lockfile + + - name: NX cache + uses: actions/cache/restore@v3 + with: + path: .nxcache + key: nx-Linux-${{ github.ref }}-${{ env.HEAD_COMMIT }} + restore-keys: ${{ env.NX_CACHE_RESTORE_KEYS }} + + - name: Build packages + run: yarn build + + - name: Get npm cache directory + id: npm-cache-dir + run: echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT + - name: Get Playwright version + id: playwright-version + run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> $GITHUB_OUTPUT + - uses: actions/cache@v3 + name: Check if Playwright browser is cached + id: playwright-cache + with: + path: ${{ steps.npm-cache-dir.outputs.dir }} + key: ${{ runner.os }}-Playwright-${{steps.playwright-version.outputs.version}} + - name: Install Playwright browser if not cached + if: steps.playwright-cache.outputs.cache-hit != 'true' + run: npx playwright install --with-deps + env: + PLAYWRIGHT_BROWSERS_PATH: ${{steps.npm-cache-dir.outputs.dir}} + - name: Install OS dependencies of Playwright if cache hit + if: steps.playwright-cache.outputs.cache-hit == 'true' + run: npx playwright install-deps + + - name: Determine changed tests + uses: getsentry/paths-filter@v2.11.1 + id: changed + with: + list-files: json + filters: | + browser_integration: packages/browser-integration-tests/suites/** + + - name: Detect flaky tests + run: yarn test:detect-flaky + working-directory: packages/browser-integration-tests + env: + CHANGED_TEST_PATHS: ${{ steps.changed.outputs.browser_integration_files }} + # Run 100 times when detecting changed test(s), else run all tests 5x + TEST_RUN_COUNT: ${{ steps.changed.outputs.browser_integration == 'true' && 100 || 5 }} diff --git a/packages/browser-integration-tests/.eslintrc.js b/packages/browser-integration-tests/.eslintrc.js index 7066f43bde55..7952587cf973 100644 --- a/packages/browser-integration-tests/.eslintrc.js +++ b/packages/browser-integration-tests/.eslintrc.js @@ -4,7 +4,7 @@ module.exports = { node: true, }, extends: ['../../.eslintrc.js'], - ignorePatterns: ['suites/**/subject.js', 'suites/**/dist/*'], + ignorePatterns: ['suites/**/subject.js', 'suites/**/dist/*', 'scripts/**'], parserOptions: { sourceType: 'module', }, diff --git a/packages/browser-integration-tests/package.json b/packages/browser-integration-tests/package.json index 88c4b70a7fb7..f9b4c34ca54a 100644 --- a/packages/browser-integration-tests/package.json +++ b/packages/browser-integration-tests/package.json @@ -28,7 +28,8 @@ "test:cjs": "PW_BUNDLE=cjs yarn test", "test:esm": "PW_BUNDLE=esm yarn test", "test:ci": "playwright test ./suites --browser='all' --reporter='line'", - "test:update-snapshots": "yarn test --update-snapshots --browser='all' && yarn test --update-snapshots" + "test:update-snapshots": "yarn test --update-snapshots --browser='all' && yarn test --update-snapshots", + "test:detect-flaky": "ts-node scripts/detectFlakyTests.ts" }, "dependencies": { "@babel/preset-typescript": "^7.16.7", @@ -40,6 +41,10 @@ "typescript": "^4.5.2", "webpack": "^5.52.0" }, + "devDependencies": { + "glob": "8.0.3", + "@types/glob": "8.0.0" + }, "volta": { "extends": "../../package.json" } diff --git a/packages/browser-integration-tests/scripts/detectFlakyTests.ts b/packages/browser-integration-tests/scripts/detectFlakyTests.ts new file mode 100644 index 000000000000..22977fa3ed83 --- /dev/null +++ b/packages/browser-integration-tests/scripts/detectFlakyTests.ts @@ -0,0 +1,80 @@ +import * as glob from 'glob'; +import * as path from 'path'; +import * as childProcess from 'child_process'; +import { promisify } from 'util'; + +const exec = promisify(childProcess.exec); + +async function run(): Promise { + let testPaths = getTestPaths(); + let failed = []; + + try { + const changedPaths: string[] = process.env.CHANGED_TEST_PATHS ? JSON.parse(process.env.CHANGED_TEST_PATHS) : []; + + if (changedPaths.length > 0) { + console.log(`Detected changed test paths: +${changedPaths.join('\n')} + +`); + + testPaths = testPaths.filter(p => changedPaths.some(changedPath => changedPath.includes(p))); + } + } catch { + console.log('Could not detect changed test paths, running all tests.'); + } + + const cwd = path.join(__dirname, '../'); + const runCount = parseInt(process.env.TEST_RUN_COUNT || '10'); + + for (const testPath of testPaths) { + console.log(`Running test: ${testPath}`); + const start = Date.now(); + + try { + await exec(`yarn playwright test ${testPath} --browser='all' --repeat-each ${runCount}`, { + cwd, + }); + const end = Date.now(); + console.log(` ☑️ Passed ${runCount} times, avg. duration ${Math.ceil((end - start) / runCount)}ms`); + } catch (error) { + logError(error); + failed.push(testPath); + } + } + + console.log(''); + console.log(''); + + if (failed.length > 0) { + console.error(`⚠️ ${failed.length} test(s) failed.`); + process.exit(1); + } else { + console.log(`☑️ ${testPaths.length} test(s) passed.`); + } +} + +function getTestPaths(): string[] { + const paths = glob.sync('suites/**/test.{ts,js}', { + cwd: path.join(__dirname, '../'), + }); + + return paths.map(p => path.dirname(p)); +} + +function logError(error: unknown) { + if (process.env.CI) { + console.log('::group::Test failed'); + } else { + console.error(' ⚠️ Test failed:'); + } + + console.log((error as any).stdout); + console.log((error as any).stderr); + + if (process.env.CI) { + console.log('::endgroup::'); + } +} + +run();