diff --git a/.github/workflows/build_reusable.yml b/.github/workflows/build_reusable.yml index 2896b81534e38..64bca82fb7173 100644 --- a/.github/workflows/build_reusable.yml +++ b/.github/workflows/build_reusable.yml @@ -95,7 +95,9 @@ env: DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }} NEXT_JUNIT_TEST_REPORT: 'true' DD_ENV: 'ci' - TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }} + # Vercel KV Store for test timings + KV_REST_API_URL: ${{ secrets.KV_REST_API_URL }} + KV_REST_API_TOKEN: ${{ secrets.KV_REST_API_TOKEN }} NEXT_TEST_JOB: 1 VERCEL_TEST_TOKEN: ${{ secrets.VERCEL_TEST_TOKEN }} VERCEL_TEST_TEAM: vtest314-next-e2e-tests diff --git a/.github/workflows/pull_request_stats.yml b/.github/workflows/pull_request_stats.yml index 14e2e76f8e09b..3b3afeee5da3b 100644 --- a/.github/workflows/pull_request_stats.yml +++ b/.github/workflows/pull_request_stats.yml @@ -16,7 +16,9 @@ env: # we build a dev binary for use in CI so skip downloading # canary next-swc binaries in the monorepo NEXT_SKIP_NATIVE_POSTINSTALL: 1 - TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }} + # Vercel KV Store for test timings + KV_REST_API_URL: ${{ secrets.KV_REST_API_URL }} + KV_REST_API_TOKEN: ${{ secrets.KV_REST_API_TOKEN }} NEXT_TEST_JOB: 1 NEXT_DISABLE_SWC_WASM: 1 diff --git a/package.json b/package.json index 8f193386a9615..504a9e143718d 100644 --- a/package.json +++ b/package.json @@ -145,7 +145,7 @@ "@typescript-eslint/eslint-plugin": "^8.36.0", "@typescript-eslint/parser": "^8.36.0", "@vercel/devlow-bench": "workspace:*", - "@vercel/fetch": "6.1.1", + "@vercel/kv": "3.0.0", "@vercel/og": "0.7.2", "abort-controller": "3.0.0", "alex": "9.1.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 367aa172875e5..77d53b7d44c9f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -223,9 +223,9 @@ importers: '@vercel/devlow-bench': specifier: workspace:* version: link:turbopack/packages/devlow-bench - '@vercel/fetch': - specifier: 6.1.1 - version: 6.1.1(@types/node-fetch@2.6.1)(node-fetch@2.6.7(encoding@0.1.13)) + '@vercel/kv': + specifier: 3.0.0 + version: 3.0.0 '@vercel/og': specifier: 0.7.2 version: 0.7.2 @@ -5454,9 +5454,6 @@ packages: '@types/aria-query@5.0.1': resolution: {integrity: sha512-XTIieEY+gvJ39ChLcB4If5zHtPxt3Syj5rgZR+e1ctpmK8NjPf0zFqsz4JpLJT0xla9GFDKjy8Cpu331nrmE1Q==} - '@types/async-retry@1.2.1': - resolution: {integrity: sha512-yMQ6CVgICWtyFNBqJT3zqOc+TnqqEPLo4nKJNPFwcialiylil38Ie6q1ENeFTjvaLOkVim9K5LisHgAKJWidGQ==} - '@types/async-retry@1.4.2': resolution: {integrity: sha512-GUDuJURF0YiJZ+CBjNQA0+vbP/VHlJbB0sFqkzsV7EcOPRfurVonXpXKAt3w8qIjM1TEzpz6hc6POocPvHOS3w==} @@ -5677,9 +5674,6 @@ packages: '@types/long@4.0.1': resolution: {integrity: sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w==} - '@types/lru-cache@4.1.1': - resolution: {integrity: sha512-8mNEUG6diOrI6pMqOHrHPDBB1JsrpedeMK9AWGzVCQ7StRRribiT9BRvUmF8aUws9iBbVlgVekOT5Sgzc1MTKw==} - '@types/mdast@3.0.10': resolution: {integrity: sha512-W864tg/Osz1+9f4lrGTZpCSO5/z4608eUp19tbozkq2HJK6i3z1kT0H9tlADXuYIb1YYOBByU4Jsqkk75q48qA==} @@ -5701,9 +5695,6 @@ packages: '@types/mute-stream@0.0.4': resolution: {integrity: sha512-CPM9nzrCPPJHQNA9keH9CVkVI+WR5kMa+7XEs5jcGQ0VoAGnLv242w8lIVgwAEfmE4oufJRaTc9PNLQl0ioAow==} - '@types/node-fetch@2.3.2': - resolution: {integrity: sha512-yW0EOebSsQme9yKu09XbdDfle4/SmWZMK4dfteWcSLCYNQQcF+YOv0kIrvm+9pO11/ghA4E6A+RNQqvYj4Nr3A==} - '@types/node-fetch@2.6.1': resolution: {integrity: sha512-oMqjURCaxoSIsHSr1E47QHzbmzNR5rK8McHuNb11BOM9cHcIK3Avy0s/b2JlXHoQGTYS3NsvWzV1M0iK7l0wbA==} @@ -6019,21 +6010,12 @@ packages: '@ungap/structured-clone@1.2.0': resolution: {integrity: sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==} - '@vercel/fetch-cached-dns@2.0.2': - resolution: {integrity: sha512-gDqKEV8CeY2YmCdZpP1rn3tFK1L07Vw2+HYkCK8zpRHOVGr/sP8yhBsW+C/yqGVj0i9z/rIvqIHe5emvRvxwgw==} - peerDependencies: - node-fetch: '*' + '@upstash/redis@1.35.3': + resolution: {integrity: sha512-hSjv66NOuahW3MisRGlSgoszU2uONAY2l5Qo3Sae8OT3/Tng9K+2/cBRuyPBX8egwEGcNNCF9+r0V6grNnhL+w==} - '@vercel/fetch-retry@5.0.3': - resolution: {integrity: sha512-DIIoBY92r+sQ6iHSf5WjKiYvkdsDIMPWKYATlE0KcUAj2RV6SZK9UWpUzBRKsofXqedOqpVjrI0IE6AWL7JRtg==} - peerDependencies: - node-fetch: '*' - - '@vercel/fetch@6.1.1': - resolution: {integrity: sha512-nddCkgpA0aVIqOlzh+qVlzDNcQq0cSnqefM+x6SciGI4GCvVZeaZ7WEowgX8I/HwBAq8Uj5Bdnd+r0+sYsJsig==} - peerDependencies: - '@types/node-fetch': '2' - node-fetch: '2' + '@vercel/kv@3.0.0': + resolution: {integrity: sha512-pKT8fRnfyYk2MgvyB6fn6ipJPCdfZwiKDdw7vB+HL50rjboEBHDVBEcnwfkEpVSp2AjNtoaOUH7zG+bVC/rvSg==} + engines: {node: '>=14.6'} '@vercel/ncc@0.34.0': resolution: {integrity: sha512-G9h5ZLBJ/V57Ou9vz5hI8pda/YQX5HQszCs3AmIus3XzsmRn/0Ptic5otD3xVST8QLKk7AMk7AqpsyQGN7MZ9A==} @@ -6153,9 +6135,6 @@ packages: '@xtuc/long@4.2.2': resolution: {integrity: sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==} - '@zeit/dns-cached-resolve@2.1.2': - resolution: {integrity: sha512-A/5gbBskKPETTBqHwvlaW1Ri2orO62yqoFoXdxna1SQ7A/lXjpWgpJ1wdY3IQEcz5LydpS4sJ8SzI2gFyyLEhg==} - JSONStream@1.3.5: resolution: {integrity: sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ==} hasBin: true @@ -6255,10 +6234,6 @@ packages: resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} engines: {node: '>= 6.0.0'} - agentkeepalive@3.4.1: - resolution: {integrity: sha512-MPIwsZU9PP9kOrZpyu2042kYA8Fdt/AedQYkYXucHgF9QoD9dXVp0ypuGnHXSR0hTstBxdt85Xkh4JolYfK5wg==} - engines: {node: '>= 4.0.0'} - agentkeepalive@4.1.4: resolution: {integrity: sha512-+V/rGa3EuU74H6wR04plBb7Ks10FbtUQgRj/FQOG7uUIEuaINI+AiqJR1k6t3SVNs7o7ZjIdus6706qqzVq8jQ==} engines: {node: '>= 8.0.0'} @@ -6769,6 +6744,7 @@ packages: binary-install@1.1.0: resolution: {integrity: sha512-rkwNGW+3aQVSZoD0/o3mfPN6Yxh3Id0R/xzTVBVVpGNlVz8EGwusksxRlbk/A5iKTZt9zkMn3qIqmAt3vpfbzg==} engines: {node: '>=10'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. bindings@1.5.0: resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} @@ -8858,6 +8834,7 @@ packages: eslint-plugin-markdown@3.0.1: resolution: {integrity: sha512-8rqoc148DWdGdmYF6WSQFT3uQ6PO7zXYgeBpHAOAakX/zpq+NvFYbDA/H7PYzHajwtmaOzAwfxyl++x0g1/N9A==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + deprecated: Please use @eslint/markdown instead peerDependencies: eslint: ^6.0.0 || ^7.0.0 || ^8.0.0 @@ -16069,6 +16046,9 @@ packages: unbox-primitive@1.0.2: resolution: {integrity: sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==} + uncrypto@0.1.3: + resolution: {integrity: sha512-Ql87qFHB3s/De2ClA9e0gsnS6zXG27SkTiSJwjCc9MebbfapQfuPzumMIUMi38ezPZVNFcHI9sUIepeQfw8J8Q==} + undici-types@6.19.8: resolution: {integrity: sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==} @@ -21377,8 +21357,6 @@ snapshots: '@types/aria-query@5.0.1': {} - '@types/async-retry@1.2.1': {} - '@types/async-retry@1.4.2': dependencies: '@types/retry': 0.12.0 @@ -21637,8 +21615,6 @@ snapshots: '@types/long@4.0.1': {} - '@types/lru-cache@4.1.1': {} - '@types/mdast@3.0.10': dependencies: '@types/unist': 2.0.3 @@ -21659,10 +21635,6 @@ snapshots: dependencies: '@types/node': 20.17.6(patch_hash=rvl3vkomen3tospgr67bzubfyu) - '@types/node-fetch@2.3.2': - dependencies: - '@types/node': 20.17.6(patch_hash=rvl3vkomen3tospgr67bzubfyu) - '@types/node-fetch@2.6.1': dependencies: '@types/node': 20.17.6(patch_hash=rvl3vkomen3tospgr67bzubfyu) @@ -22102,31 +22074,13 @@ snapshots: '@ungap/structured-clone@1.2.0': {} - '@vercel/fetch-cached-dns@2.0.2(node-fetch@2.6.7(encoding@0.1.13))': + '@upstash/redis@1.35.3': dependencies: - '@types/node-fetch': 2.3.2 - '@zeit/dns-cached-resolve': 2.1.2 - node-fetch: 2.6.7(encoding@0.1.13) + uncrypto: 0.1.3 - '@vercel/fetch-retry@5.0.3(node-fetch@2.6.7(encoding@0.1.13))': + '@vercel/kv@3.0.0': dependencies: - async-retry: 1.3.1 - debug: 3.2.7 - node-fetch: 2.6.7(encoding@0.1.13) - transitivePeerDependencies: - - supports-color - - '@vercel/fetch@6.1.1(@types/node-fetch@2.6.1)(node-fetch@2.6.7(encoding@0.1.13))': - dependencies: - '@types/async-retry': 1.2.1 - '@types/node-fetch': 2.6.1 - '@vercel/fetch-cached-dns': 2.0.2(node-fetch@2.6.7(encoding@0.1.13)) - '@vercel/fetch-retry': 5.0.3(node-fetch@2.6.7(encoding@0.1.13)) - agentkeepalive: 3.4.1 - debug: 3.1.0 - node-fetch: 2.6.7(encoding@0.1.13) - transitivePeerDependencies: - - supports-color + '@upstash/redis': 1.35.3 '@vercel/ncc@0.34.0': {} @@ -22320,14 +22274,6 @@ snapshots: '@xtuc/long@4.2.2': {} - '@zeit/dns-cached-resolve@2.1.2': - dependencies: - '@types/async-retry': 1.2.1 - '@types/lru-cache': 4.1.1 - '@types/node': 20.17.6(patch_hash=rvl3vkomen3tospgr67bzubfyu) - async-retry: 1.2.3 - lru-cache: 5.1.1 - JSONStream@1.3.5: dependencies: jsonparse: 1.3.1 @@ -22412,10 +22358,6 @@ snapshots: transitivePeerDependencies: - supports-color - agentkeepalive@3.4.1: - dependencies: - humanize-ms: 1.2.1 - agentkeepalive@4.1.4: dependencies: debug: 4.1.1 @@ -34722,6 +34664,8 @@ snapshots: has-symbols: 1.0.3 which-boxed-primitive: 1.0.2 + uncrypto@0.1.3: {} + undici-types@6.19.8: {} undici@5.26.3: diff --git a/run-tests.js b/run-tests.js index 38c43f70da8f2..108b0bed18ee5 100644 --- a/run-tests.js +++ b/run-tests.js @@ -4,10 +4,7 @@ const path = require('path') const _glob = require('glob') const { existsSync } = require('fs') const fsp = require('fs/promises') -const nodeFetch = require('node-fetch') -const vercelFetch = require('@vercel/fetch') -// @ts-expect-error -const fetch = vercelFetch(nodeFetch) +const { createClient } = require('@vercel/kv') const { promisify } = require('util') const { Sema } = require('async-sema') const { spawn, exec: execOrig } = require('child_process') @@ -65,14 +62,47 @@ const shouldContinueTestsOnError = !!process.env.NEXT_TEST_CONTINUE_ON_ERROR const skipRetryTestManifest = process.env.NEXT_TEST_SKIP_RETRY_MANIFEST ? require(process.env.NEXT_TEST_SKIP_RETRY_MANIFEST) : [] -const TIMINGS_API = `https://api.github.com/gists/4500dd89ae2f5d70d9aaceb191f528d1` -const TIMINGS_API_HEADERS = { - Accept: 'application/vnd.github.v3+json', - ...(process.env.TEST_TIMINGS_TOKEN - ? { - Authorization: `Bearer ${process.env.TEST_TIMINGS_TOKEN}`, +const KV_TIMINGS_KEY = 'test-timings' + +const kvClient = + process.env.KV_REST_API_URL && process.env.KV_REST_API_TOKEN + ? createClient({ + url: process.env.KV_REST_API_URL, + token: process.env.KV_REST_API_TOKEN, + }) + : null + +/** + * Retry a KV operation with exponential backoff + * @param {() => Promise} operation - The async operation to retry + * @param {string} operationName - Name of the operation for logging + * @param {number} maxRetries - Maximum number of retries (default: 3) + * @returns {Promise} The result of the operation + */ +async function retryKVOperation(operation, operationName, maxRetries = 3) { + let lastError + let retries = maxRetries + + while (retries > 0) { + try { + return await operation() + } catch (err) { + lastError = err + retries-- + if (retries > 0) { + const delay = (maxRetries - retries + 1) * 5 // 5s, 10s, 15s backoff + console.log( + `KV ${operationName} failed, retrying in ${delay}s. Error:`, + err.message + ) + await new Promise((resolve) => setTimeout(resolve, delay * 1000)) } - : {}), + } + } + + throw new Error( + `Failed to ${operationName} after ${maxRetries} retries: ${lastError?.message}` + ) } const testFilters = { @@ -168,28 +198,19 @@ const isMatchingPattern = (pattern, file) => { } async function getTestTimings() { - let timingsRes - - const doFetch = () => - fetch(TIMINGS_API, { - headers: { - ...TIMINGS_API_HEADERS, - }, - }) - timingsRes = await doFetch() - - if (timingsRes.status === 403) { - const delay = 15 - console.log(`Got 403 response waiting ${delay} seconds before retry`) - await new Promise((resolve) => setTimeout(resolve, delay * 1000)) - timingsRes = await doFetch() + if (!kvClient) { + console.warn('KV client not configured, skipping timing fetch') + return null } - if (!timingsRes.ok) { - throw new Error(`request status: ${timingsRes.status}`) - } - const timingsData = await timingsRes.json() - return JSON.parse(timingsData.files['test-timings.json'].content) + const timings = await retryKVOperation(async () => { + const data = await kvClient.get(KV_TIMINGS_KEY) + if (!data) { + console.log('No timing data found in KV store') + } + return data + }, 'fetch timings') + return timings || null } async function main() { @@ -243,6 +264,10 @@ async function main() { process.env.NEXT_TEST_MODE ) + // Only fetch/update shared timing data during grouped CI runs to avoid + // individual test runs from polluting the timing data + const shouldUseSharedTimings = options.timings && options.group + /** @type TestFile[] */ let tests = argv._.filter((arg) => arg.toString().match(/\.test\.(js|ts|tsx)/) @@ -302,30 +327,45 @@ async function main() { // } - if (options.timings && options.group) { + if (shouldUseSharedTimings) { console.log('Fetching previous timings data') + const timingsFile = path.join(process.cwd(), 'test-timings.json') + try { - const timingsFile = path.join(process.cwd(), 'test-timings.json') - try { - prevTimings = JSON.parse(await fsp.readFile(timingsFile, 'utf8')) - console.log('Loaded test timings from disk successfully') - } catch (_) { - console.error('failed to load from disk', _) - } + prevTimings = JSON.parse(await fsp.readFile(timingsFile, 'utf8')) + console.log('Loaded test timings from disk successfully') + } catch (_) { + console.error( + 'Failed to load test timings from disk. Proceeding to fetch from KV store. Original error: ', + _ + ) + } - if (!prevTimings) { + if (!prevTimings) { + try { prevTimings = await getTestTimings() - console.log('Fetched previous timings data successfully') + if (prevTimings) { + console.log('Fetched previous timings data successfully from KV') + } else { + console.log('No previous timings data available') + } + } catch (kvError) { + console.warn( + 'Failed to fetch timings from KV, continuing without timing data:', + kvError.message + ) + prevTimings = null + } - if (options.writeTimings) { + if (options.writeTimings) { + if (prevTimings) { await fsp.writeFile(timingsFile, JSON.stringify(prevTimings)) console.log('Wrote previous timings data to', timingsFile) - await cleanUpAndExit(0) + } else { + console.log('No timings data to write') } + await cleanUpAndExit(0) } - } catch (err) { - console.log(`Failed to fetch timings data`, err) - await cleanUpAndExit(1) } } @@ -389,6 +429,14 @@ async function main() { // tests tend not to get clustered together tests = tests.filter((_value, idx) => idx % groupTotal === groupPos - 1) console.log('Splitting without timings') + + // Warn in CI that tests are not optimally distributed + if (process.env.GITHUB_ACTIONS) { + core.warning( + `Test timing data unavailable for group ${options.group}. Tests are being distributed round-robin, which may increase CI time. ` + + `Consider checking KV store connectivity if this persists.` + ) + } } } @@ -774,43 +822,34 @@ ${ENDGROUP}`) // junitData += `` // console.log('output timing data to junit.xml') - if (prevTimings && process.env.TEST_TIMINGS_TOKEN) { - try { - const newTimings = { - ...(await getTestTimings()), - ...curTimings, - } + if (shouldUseSharedTimings) { + if (kvClient) { + try { + // Fetch existing timings and merge with new ones + const existingTimings = (await getTestTimings()) || {} + const newTimings = { + ...existingTimings, + ...curTimings, + } - for (const test of Object.keys(newTimings)) { - if (!existsSync(path.join(__dirname, test))) { - console.log('removing stale timing', test) - delete newTimings[test] + // Clean up stale timings for deleted tests + for (const test of Object.keys(newTimings)) { + if (!existsSync(path.join(__dirname, test))) { + console.log('removing stale timing', test) + delete newTimings[test] + } } - } - const timingsRes = await fetch(TIMINGS_API, { - method: 'PATCH', - headers: { - ...TIMINGS_API_HEADERS, - }, - body: JSON.stringify({ - files: { - 'test-timings.json': { - content: JSON.stringify(newTimings), - }, - }, - }), - }) - - if (!timingsRes.ok) { - throw new Error(`request status: ${timingsRes.status}`) + // Update KV store with retries + await retryKVOperation(async () => { + await kvClient.set(KV_TIMINGS_KEY, newTimings) + console.log('Successfully updated test timings in KV store') + }, 'update timings') + } catch (err) { + console.log('Failed to update timings data', err) } - const result = await timingsRes.json() - console.log( - `Sent updated timings successfully. API URL: "${result?.url}" HTML URL: "${result?.html_url}"` - ) - } catch (err) { - console.log('Failed to update timings data', err) + } else { + console.warn('KV client not configured, skipping timing update') } } } diff --git a/turbo.json b/turbo.json index 8c87b7671c847..b7d5d171d1c2b 100644 --- a/turbo.json +++ b/turbo.json @@ -23,7 +23,8 @@ "//#typescript": {}, "//#get-test-timings": { "inputs": ["run-tests.js"], - "outputs": ["test-timings.json"] + "outputs": ["test-timings.json"], + "env": ["KV_REST_API_URL", "KV_REST_API_TOKEN"] } }, "ui": "tui"