ci(replay): Overhead measurement

.github/workflows/build.yml

@@ -815,3 +815,51 @@ jobs:
         if: contains(needs.*.result, 'failure')
         run: |
           echo "One of the dependent jobs have failed. You may need to re-run it." && exit 1
+
+  replay_metrics:
+    name: Replay Metrics
+    needs: [job_get_metadata, job_build]
+    runs-on: ubuntu-20.04
+    timeout-minutes: 30
+    if: contains(github.event.pull_request.labels.*.name, 'ci-overhead-measurements')
+    steps:
+      - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ env.HEAD_COMMIT }}
+      - name: Set up Node
+        uses: volta-cli/action@v4
+      - name: Check dependency cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.CACHED_DEPENDENCY_PATHS }}
+          key: ${{ needs.job_build.outputs.dependency_cache_key }}
+      - name: Check build cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.CACHED_BUILD_PATHS }}
+          key: ${{ env.BUILD_CACHE_KEY }}
+
+      - name: Setup
+        run: yarn install
+        working-directory: packages/replay/metrics
+
+      - name: Collect
+        run: yarn ci:collect
+        working-directory: packages/replay/metrics
+
+      - name: Process
+        id: process
+        run: yarn ci:process
+        working-directory: packages/replay/metrics
+        # Don't run on forks - the PR comment cannot be added.
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+
+      - name: Upload results
+        uses: actions/upload-artifact@v3
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+        with:
+          name: ${{ steps.process.outputs.artifactName }}
+          path: ${{ steps.process.outputs.artifactPath }}

.gitignore

@@ -47,4 +47,4 @@ tmp.js
 
 # eslint
 .eslintcache
-eslintcache/*
+**/eslintcache/*

.vscode/launch.json

@@ -37,6 +37,22 @@
       "internalConsoleOptions": "openOnSessionStart",
       "outputCapture": "std"
     },
+    {
+        "type": "node",
+        "name": "Debug replay metrics collection script",
+        "request": "launch",
+        "cwd": "${workspaceFolder}/packages/replay/metrics/",
+        "program": "${workspaceFolder}/packages/replay/metrics/configs/dev/collect.ts",
+        "preLaunchTask": "Build Replay metrics script",
+    },
+    {
+        "type": "node",
+        "name": "Debug replay metrics processing script",
+        "request": "launch",
+        "cwd": "${workspaceFolder}/packages/replay/metrics/",
+        "program": "${workspaceFolder}/packages/replay/metrics/configs/dev/process.ts",
+        "preLaunchTask": "Build Replay metrics script",
+    },
     // Run rollup using the config file which is in the currently active tab.
     {
       "name": "Debug rollup (config from open file)",

.vscode/tasks.json

@@ -7,7 +7,13 @@
 			"type": "npm",
 			"script": "predebug",
 			"path": "packages/nextjs/test/integration/",
-			"detail": "Link the SDK (if not already linked) and build test app"
+			"detail": "Link the SDK (if not already linked) and build test app",
+		},
+		{
+			"label": "Build Replay metrics script",
+			"type": "npm",
+			"script": "build",
+			"path": "packages/replay/metrics",
 		}
 	]
 }

packages/replay/.eslintignore

@@ -3,3 +3,4 @@ build/
 demo/build/
 # TODO: Check if we can re-introduce linting in demo
 demo
+metrics

packages/replay/metrics/.eslintrc.cjs

@@ -0,0 +1,14 @@
+module.exports = {
+  extends: ['../.eslintrc.js'],
+  ignorePatterns: ['test-apps'],
+  overrides: [
+    {
+      files: ['*.ts'],
+      rules: {
+        'no-console': 'off',
+        '@typescript-eslint/no-non-null-assertion': 'off',
+        'import/no-unresolved': 'off',
+      },
+    },
+  ],
+};

packages/replay/metrics/.gitignore

@@ -0,0 +1 @@
+out

packages/replay/metrics/README.md

@@ -0,0 +1,11 @@
+# Replay performance metrics
+
+Evaluates Replay impact on website performance by running a web app in Chromium via Playwright and collecting various metrics.
+
+The general idea is to run a web app without Sentry Replay and then run the same app again with Sentry and another one with Sentry+Replay included.
+For the three scenarios, we collect some metrics (CPU, memory, vitals) and later compare them and post as a comment in a PR.
+Changes in the metrics, compared to previous runs from the main branch, should be evaluated on case-by-case basis when preparing and reviewing the PR.
+
+## Resources
+
+* https://github.com/addyosmani/puppeteer-webperf

packages/replay/metrics/configs/README.md

@@ -0,0 +1,4 @@
+# Replay metrics configuration & entrypoints (scripts)
+
+* [dev](dev) contains scripts launched during local development
+* [ci](ci) contains scripts launched in CI

packages/replay/metrics/configs/ci/collect.ts

@@ -0,0 +1,54 @@
+import { Metrics, MetricsCollector } from '../../src/collector.js';
+import { MetricsStats, NumberProvider } from '../../src/results/metrics-stats.js';
+import { JankTestScenario } from '../../src/scenarios.js';
+import { printStats } from '../../src/util/console.js';
+import { latestResultFile } from './env.js';
+
+function checkStdDev(results: Metrics[], name: string, provider: NumberProvider, max: number): boolean {
+  const value = MetricsStats.stddev(results, provider);
+  if (value == undefined) {
+    console.warn(`✗ | Discarding results because StandardDeviation(${name}) is undefined`);
+    return false;
+  } else if (value > max) {
+    console.warn(`✗ | Discarding results because StandardDeviation(${name}) is larger than ${max}. Actual value: ${value}`);
+    return false;
+  } else {
+    console.log(`✓ | StandardDeviation(${name}) is ${value} (<= ${max})`)
+  }
+  return true;
+}
+
+const collector = new MetricsCollector({ headless: true, cpuThrottling: 2 });
+const result = await collector.execute({
+  name: 'jank',
+  scenarios: [
+    new JankTestScenario('index.html'),
+    new JankTestScenario('with-sentry.html'),
+    new JankTestScenario('with-replay.html'),
+  ],
+  runs: 10,
+  tries: 10,
+  async shouldAccept(results: Metrics[]): Promise<boolean> {
+    await printStats(results);
+
+    if (!checkStdDev(results, 'lcp', MetricsStats.lcp, 50)
+      || !checkStdDev(results, 'cls', MetricsStats.cls, 0.1)
+      || !checkStdDev(results, 'cpu', MetricsStats.cpu, 1)
+      || !checkStdDev(results, 'memory-mean', MetricsStats.memoryMean, 1000 * 1024)
+      || !checkStdDev(results, 'memory-max', MetricsStats.memoryMax, 1000 * 1024)) {
+      return false;
+    }
+
+    const cpuUsage = MetricsStats.mean(results, MetricsStats.cpu)!;
+    if (cpuUsage > 0.85) {
+      // Note: complexity on the "JankTest" is defined by the `minimum = ...,` setting in app.js - specifying the number of animated elements.
+      console.warn(`✗ | Discarding results because CPU usage is too high and may be inaccurate: ${(cpuUsage * 100).toFixed(2)} %.`,
+        'Consider simplifying the scenario or changing the CPU throttling factor.');
+      return false;
+    }
+
+    return true;
+  },
+});
+
+result.writeToFile(latestResultFile);

packages/replay/metrics/configs/ci/env.ts

@@ -0,0 +1,4 @@
+export const previousResultsDir = 'out/previous-results';
+export const baselineResultsDir = 'out/baseline-results';
+export const latestResultFile = 'out/latest-result.json';
+export const artifactName = 'replay-sdk-metrics'

packages/replay/metrics/configs/ci/process.ts

@@ -0,0 +1,44 @@
+import path from 'path';
+
+import { ResultsAnalyzer } from '../../src/results/analyzer.js';
+import { PrCommentBuilder } from '../../src/results/pr-comment.js';
+import { Result } from '../../src/results/result.js';
+import { ResultsSet } from '../../src/results/results-set.js';
+import { Git } from '../../src/util/git.js';
+import { GitHub } from '../../src/util/github.js';
+import { artifactName, baselineResultsDir, latestResultFile, previousResultsDir } from './env.js';
+
+const latestResult = Result.readFromFile(latestResultFile);
+const branch = await Git.branch;
+const baseBranch = await Git.baseBranch;
+
+await GitHub.downloadPreviousArtifact(baseBranch, baselineResultsDir, artifactName);
+await GitHub.downloadPreviousArtifact(branch, previousResultsDir, artifactName);
+
+GitHub.writeOutput('artifactName', artifactName)
+GitHub.writeOutput('artifactPath', path.resolve(previousResultsDir));
+
+const previousResults = new ResultsSet(previousResultsDir);
+
+const prComment = new PrCommentBuilder();
+if (baseBranch != branch) {
+  const baseResults = new ResultsSet(baselineResultsDir);
+  await prComment.addCurrentResult(await ResultsAnalyzer.analyze(latestResult, baseResults), 'Baseline');
+  await prComment.addAdditionalResultsSet(
+    `Baseline results on branch: <code>${baseBranch}</code>`,
+    // We skip the first one here because it's already included as `Baseline` column above in addCurrentResult().
+    baseResults.items().slice(1, 10)
+  );
+} else {
+  await prComment.addCurrentResult(await ResultsAnalyzer.analyze(latestResult, previousResults), 'Previous');
+}
+
+await prComment.addAdditionalResultsSet(
+  `Previous results on branch: <code>${branch}</code>`,
+  previousResults.items().slice(0, 10)
+);
+
+await GitHub.addOrUpdateComment(prComment);
+
+// Copy the latest test run results to the archived result dir.
+await previousResults.add(latestResultFile, true);

packages/replay/metrics/configs/dev/collect.ts

@@ -0,0 +1,30 @@
+import { Metrics, MetricsCollector } from '../../src/collector.js';
+import { MetricsStats } from '../../src/results/metrics-stats.js';
+import { JankTestScenario } from '../../src/scenarios.js';
+import { printStats } from '../../src/util/console.js';
+import { latestResultFile } from './env.js';
+
+const collector = new MetricsCollector();
+const result = await collector.execute({
+  name: 'dummy',
+  scenarios: [
+    new JankTestScenario('index.html'),
+    new JankTestScenario('with-sentry.html'),
+    new JankTestScenario('with-replay.html'),
+  ],
+  runs: 1,
+  tries: 1,
+  async shouldAccept(results: Metrics[]): Promise<boolean> {
+    printStats(results);
+
+    const cpuUsage = MetricsStats.mean(results, MetricsStats.cpu)!;
+    if (cpuUsage > 0.9) {
+      console.error(`CPU usage too high to be accurate: ${(cpuUsage * 100).toFixed(2)} %.`,
+        'Consider simplifying the scenario or changing the CPU throttling factor.');
+      return false;
+    }
+    return true;
+  },
+});
+
+result.writeToFile(latestResultFile);

packages/replay/metrics/configs/dev/env.ts

@@ -0,0 +1,2 @@
+export const outDir = 'out/results-dev';
+export const latestResultFile = 'out/latest-result.json';

packages/replay/metrics/configs/dev/process.ts

@@ -0,0 +1,13 @@
+import { ResultsAnalyzer } from '../../src/results/analyzer.js';
+import { Result } from '../../src/results/result.js';
+import { ResultsSet } from '../../src/results/results-set.js';
+import { printAnalysis } from '../../src/util/console.js';
+import { latestResultFile, outDir } from './env.js';
+
+const resultsSet = new ResultsSet(outDir);
+const latestResult = Result.readFromFile(latestResultFile);
+
+const analysis = await ResultsAnalyzer.analyze(latestResult, resultsSet);
+printAnalysis(analysis);
+
+await resultsSet.add(latestResultFile, true);

packages/replay/metrics/package.json

@@ -0,0 +1,32 @@
+{
+  "private": true,
+  "name": "metrics",
+  "main": "index.js",
+  "author": "Sentry",
+  "license": "MIT",
+  "type": "module",
+  "scripts": {
+    "build": "tsc",
+    "deps": "yarn --cwd ../ build:bundle && yarn --cwd ../../tracing/ build:bundle",
+    "dev:collect": "ts-node-esm ./configs/dev/collect.ts",
+    "dev:process": "ts-node-esm ./configs/dev/process.ts",
+    "ci:collect": "ts-node-esm ./configs/ci/collect.ts",
+    "ci:process": "ts-node-esm ./configs/ci/process.ts"
+  },
+  "dependencies": {
+    "@octokit/rest": "^19.0.5",
+    "@types/node": "^18.11.17",
+    "axios": "^1.2.2",
+    "extract-zip": "^2.0.1",
+    "filesize": "^10.0.6",
+    "p-timeout": "^6.0.0",
+    "playwright": "^1.29.1",
+    "playwright-core": "^1.29.1",
+    "simple-git": "^3.15.1",
+    "simple-statistics": "^7.8.0",
+    "typescript": "^4.9.4"
+  },
+  "devDependencies": {
+    "ts-node": "^10.9.1"
+  }
+}