ci: Add workflow to detect flaky tests

.github/workflows/flaky-test-detector.yml

@@ -0,0 +1,81 @@
+name: 'Detect flaky tests'
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - 'packages/browser-integration-tests/suites/**'
+
+env:
+  HEAD_COMMIT: ${{ github.event.inputs.commit || github.sha }}
+
+  NX_CACHE_RESTORE_KEYS: |
+    nx-Linux-${{ github.ref }}-${{ github.event.inputs.commit || github.sha }}
+    nx-Linux-${{ github.ref }}
+    nx-Linux
+
+# Cancel in progress workflows on pull_requests.
+# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  flaky-detector:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 60
+    name: 'Check tests for flakiness'
+    steps:
+      - name: Check out current branch
+        uses: actions/checkout@v3
+      - name: Set up Node
+        uses: volta-cli/action@v4
+
+      - name: Install dependencies
+        run: yarn install --ignore-engines --frozen-lockfile
+
+      - name: NX cache
+        uses: actions/cache/restore@v3
+        with:
+          path: .nxcache
+          key: nx-Linux-${{ github.ref }}-${{ env.HEAD_COMMIT }}
+          restore-keys: ${{ env.NX_CACHE_RESTORE_KEYS }}
+
+      - name: Build packages
+        run: yarn build
+
+      - name: Get npm cache directory
+        id: npm-cache-dir
+        run: echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT
+      - name: Get Playwright version
+        id: playwright-version
+        run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> $GITHUB_OUTPUT
+      - uses: actions/cache@v3
+        name: Check if Playwright browser is cached
+        id: playwright-cache
+        with:
+          path: ${{ steps.npm-cache-dir.outputs.dir }}
+          key: ${{ runner.os }}-Playwright-${{steps.playwright-version.outputs.version}}
+      - name: Install Playwright browser if not cached
+        if: steps.playwright-cache.outputs.cache-hit != 'true'
+        run: npx playwright install --with-deps
+        env:
+          PLAYWRIGHT_BROWSERS_PATH: ${{steps.npm-cache-dir.outputs.dir}}
+      - name: Install OS dependencies of Playwright if cache hit
+        if: steps.playwright-cache.outputs.cache-hit == 'true'
+        run: npx playwright install-deps
+
+      - name: Determine changed tests
+        uses: getsentry/[email protected]
+        id: changed
+        with:
+          list-files: json
+          filters: |
+            browser_integration: packages/browser-integration-tests/suites/**
+
+      - name: Detect flaky tests
+        run: yarn test:detect-flaky
+        working-directory: packages/browser-integration-tests
+        env:
+          CHANGED_TEST_PATHS: ${{ steps.changed.outputs.browser_integration_files }}
+          # Run 100 times when detecting changed test(s), else run all tests 5x
+          TEST_RUN_COUNT: ${{ steps.changed.outputs.browser_integration == 'true' && 100 || 5 }}

packages/browser-integration-tests/.eslintrc.js

@@ -4,7 +4,7 @@ module.exports = {
     node: true,
   },
   extends: ['../../.eslintrc.js'],
-  ignorePatterns: ['suites/**/subject.js', 'suites/**/dist/*'],
+  ignorePatterns: ['suites/**/subject.js', 'suites/**/dist/*', 'scripts/**'],
   parserOptions: {
     sourceType: 'module',
   },

packages/browser-integration-tests/package.json

@@ -28,7 +28,8 @@
     "test:cjs": "PW_BUNDLE=cjs yarn test",
     "test:esm": "PW_BUNDLE=esm yarn test",
     "test:ci": "playwright test ./suites --browser='all' --reporter='line'",
-    "test:update-snapshots": "yarn test --update-snapshots --browser='all' && yarn test --update-snapshots"
+    "test:update-snapshots": "yarn test --update-snapshots --browser='all' && yarn test --update-snapshots",
+    "test:detect-flaky": "ts-node scripts/detectFlakyTests.ts"
   },
   "dependencies": {
     "@babel/preset-typescript": "^7.16.7",
@@ -40,6 +41,10 @@
     "typescript": "^4.5.2",
     "webpack": "^5.52.0"
   },
+  "devDependencies": {
+    "glob": "8.0.3",
+    "@types/glob": "8.0.0"
+  },
   "volta": {
     "extends": "../../package.json"
   }

packages/browser-integration-tests/scripts/detectFlakyTests.ts

@@ -0,0 +1,80 @@
+import * as glob from 'glob';
+import * as path from 'path';
+import * as childProcess from 'child_process';
+import { promisify } from 'util';
+
+const exec = promisify(childProcess.exec);
+
+async function run(): Promise<void> {
+  let testPaths = getTestPaths();
+  let failed = [];
+
+  try {
+    const changedPaths: string[] = process.env.CHANGED_TEST_PATHS ? JSON.parse(process.env.CHANGED_TEST_PATHS) : [];
+
+    if (changedPaths.length > 0) {
+      console.log(`Detected changed test paths:
+${changedPaths.join('\n')}
+
+`);
+
+      testPaths = testPaths.filter(p => changedPaths.some(changedPath => changedPath.includes(p)));
+    }
+  } catch {
+    console.log('Could not detect changed test paths, running all tests.');
+  }
+
+  const cwd = path.join(__dirname, '../');
+  const runCount = parseInt(process.env.TEST_RUN_COUNT || '10');
+
+  for (const testPath of testPaths) {
+    console.log(`Running test: ${testPath}`);
+    const start = Date.now();
+
+    try {
+      await exec(`yarn playwright test ${testPath} --browser='all' --repeat-each ${runCount}`, {
+        cwd,
+      });
+      const end = Date.now();
+      console.log(`  ☑️  Passed ${runCount} times, avg. duration ${Math.ceil((end - start) / runCount)}ms`);
+    } catch (error) {
+      logError(error);
+      failed.push(testPath);
+    }
+  }
+
+  console.log('');
+  console.log('');
+
+  if (failed.length > 0) {
+    console.error(`⚠️ ${failed.length} test(s) failed.`);
+    process.exit(1);
+  } else {
+    console.log(`☑️ ${testPaths.length} test(s) passed.`);
+  }
+}
+
+function getTestPaths(): string[] {
+  const paths = glob.sync('suites/**/test.{ts,js}', {
+    cwd: path.join(__dirname, '../'),
+  });
+
+  return paths.map(p => path.dirname(p));
+}
+
+function logError(error: unknown) {
+  if (process.env.CI) {
+    console.log('::group::Test failed');
+  } else {
+    console.error(' ⚠️ Test failed:');
+  }
+
+  console.log((error as any).stdout);
+  console.log((error as any).stderr);
+
+  if (process.env.CI) {
+    console.log('::endgroup::');
+  }
+}
+
+run();