chore(repo): Cleanup leftover users from the e2e instances (#2181)

* chore(repo): Cleanup leftover users from the e2e instances

Playwright will not run the cleanup tasks on test failure, so it's possible that some temp users will not be deleted.

* Create moody-radios-begin.md

Author: nikosdouvlis

.changeset/moody-radios-begin.md

@@ -0,0 +1,2 @@
+---
+---

.github/workflows/e2e-cleanups.yml

@@ -0,0 +1,42 @@
+name: Nightly upstream tests
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 5,17 * * *'
+
+jobs:
+  integration-tests:
+    name: Cleanup e2e instances
+    runs-on: ${{ vars.RUNNER_NORMAL }}
+    timeout-minutes: ${{ fromJSON(vars.TIMEOUT_MINUTES_NORMAL) }}
+
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          show-progress: false
+
+      - name: Setup
+        id: config
+        uses: ./.github/actions/init
+        with:
+          turbo-signature: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY }}
+          turbo-team: ${{ vars.TURBO_TEAM }}
+          turbo-token: ${{ secrets.TURBO_TOKEN }}
+          playwright-enabled: true
+
+      - name: Verdaccio
+        uses: ./.github/actions/verdaccio
+        with:
+          publish-cmd: |
+            if [ "$(npm config get registry)" = "https://registry.npmjs.org/" ]; then echo 'Error: Using default registry' && exit 1; else npx turbo build $TURBO_ARGS && npx changeset publish --no-git-tag; fi
+
+      - name: Install @clerk/backend in /integration
+        working-directory: ./integration
+        run: npm init -y && npm install @clerk/backend
+
+      - name: Run cleanup
+        run: npm run test:integration:cleanup
+        env:
+          INTEGRATION_INSTANCE_KEYS: ${{ secrets.INTEGRATION_INSTANCE_KEYS }}

integration/cleanup/cleanup.setup.ts

@@ -0,0 +1,58 @@
+import type { User } from '@clerk/backend';
+import { Clerk } from '@clerk/backend';
+import { test as setup } from '@playwright/test';
+
+import { appConfigs } from '../presets/';
+
+setup('cleanup instances ', async () => {
+  const secretKeys = Object.values(appConfigs.envs)
+    .map(e => e.toJson())
+    .map(json => json.private)
+    .map(keys => keys['CLERK_SECRET_KEY'])
+    .filter(Boolean);
+
+  for (const secretKey of secretKeys) {
+    console.log(`Cleanup for ${secretKey.replace(/(sk_test_)(.+)(...)/, '$1***$3')}`);
+    const clerkClient = Clerk({ secretKey });
+    const { data: users, errors } = await clerkClient.users.getUserList({
+      orderBy: '-created_at',
+      query: 'clerkcookie',
+      limit: 100,
+    });
+
+    if (errors) {
+      console.log(errors);
+      return;
+    }
+
+    const batches = batchElements(skipUsersThatWereCreatedToday(users), 5);
+    for (const batch of batches) {
+      console.log(`Starting batch...`);
+      await Promise.all(
+        batch.map(user => {
+          console.log(
+            `Cleaning up user ${user.id} (${user.emailAddresses[0]?.emailAddress}) (${new Date(
+              user.createdAt,
+            ).toISOString()})`,
+          );
+          return clerkClient.users.deleteUser(user.id);
+        }),
+      );
+      await new Promise(r => setTimeout(r, 1000));
+    }
+  }
+});
+
+const skipUsersThatWereCreatedToday = (users: User[]): User[] => {
+  const today = new Date();
+  const todayString = today.toISOString().slice(0, 10);
+  return users.filter(user => new Date(user.createdAt).toISOString().slice(0, 10) !== todayString);
+};
+
+function batchElements<T>(users: T[], batchSize = 5): T[][] {
+  const batches = [];
+  for (let i = 0; i < users.length; i += batchSize) {
+    batches.push(users.slice(i, i + batchSize));
+  }
+  return batches;
+}

integration/playwright.cleanup.config.ts

@@ -0,0 +1,18 @@
+import { defineConfig } from '@playwright/test';
+import { config } from 'dotenv';
+import * as path from 'path';
+
+import { common } from './playwright.config';
+
+config({ path: path.resolve(__dirname, '.env.local') });
+
+export default defineConfig({
+  ...common,
+  testDir: './cleanup',
+  projects: [
+    {
+      name: 'setup',
+      testMatch: /cleanup\.setup/,
+    },
+  ],
+});

package.json

@@ -31,8 +31,9 @@
     "test": "FORCE_COLOR=1 turbo test --concurrency=${TURBO_CONCURRENCY:-80%}",
     "test:cache:clear": "FORCE_COLOR=1 turbo test:cache:clear --continue --concurrency=${TURBO_CONCURRENCY:-80%}",
     "test:integration:base": "DEBUG=1 npx playwright test --config integration/playwright.config.ts",
+    "test:integration:cleanup": "DEBUG=1 npx playwright test --config integration/playwright.cleanup.config.ts",
     "test:integration:deployment:nextjs": "DEBUG=1 npx playwright test --config integration/playwright.deployments.config.ts",
-    "test:integration:express": "E2E_APP_ID=express.* npm run test:integration:base -- --grep \"@generic|@express\"",
+    "test:integration:express": "E2E_APP_ID=express.* npm run test:integration:base -- --grep @express",
     "test:integration:generic": "E2E_APP_ID=react.vite.* npm run test:integration:base -- --grep @generic",
     "test:integration:nextjs": "E2E_APP_ID=next.appRouter.withEmailCodes npm run test:integration:base -- --grep \"@generic|@nextjs\"",
     "test:integration:remix": "echo 'placeholder'",