test: fetch-retry for github actions

Author: pieh

.github/workflows/run-tests.yml

@@ -124,6 +124,7 @@ jobs:
           NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
           NEXT_VERSION: ${{ matrix.version }}
           NEXT_RESOLVED_VERSION: ${{ steps.resolve-next-version.outputs.version }}
+          NODE_OPTIONS: --import ${{ github.workspace }}/tools/fetch-retry.mjs
       - name: Upload blob report to GitHub Actions Artifacts
         uses: actions/upload-artifact@v4
         if: always()
@@ -220,13 +221,15 @@ jobs:
         env:
           NEXT_VERSION: ${{ matrix.version }}
           NEXT_RESOLVED_VERSION: ${{ steps.resolve-next-version.outputs.version }}
+          NODE_OPTIONS: --import ${{ github.workspace }}/tools/fetch-retry.mjs
       - name: 'Unit and integration tests'
         run: npm run test:ci:unit-and-integration -- --shard=${{ matrix.shard }}/8
         env:
           NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
           NEXT_VERSION: ${{ matrix.version }}
           NEXT_RESOLVED_VERSION: ${{ steps.resolve-next-version.outputs.version }}
           TEMP: ${{ github.workspace }}/..
+          NODE_OPTIONS: --import ${{ github.workspace }}/tools/fetch-retry.mjs
 
   smoke:
     if: always()
@@ -289,6 +292,7 @@ jobs:
           NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
           NEXT_VERSION: ${{ matrix.version }}
           NEXT_RESOLVED_VERSION: ${{ steps.resolve-next-version.outputs.version }}
+          NODE_OPTIONS: --import ${{ github.workspace }}/tools/fetch-retry.mjs
 
   merge-reports:
     if: always()

tools/fetch-retry.mjs

@@ -0,0 +1,41 @@
+// We are seeing quite a bit of 'fetch failed' cases in Github Actions that don't really reproduce
+// locally. We are likely hitting some limits there when attempting to parallelize. They are not consistent
+// so instead of reducing parallelism, we add a retry with backoff here.
+
+const originalFetch = globalThis.fetch
+
+const NUM_RETRIES = 5
+
+globalThis.fetch = async (...args) => {
+  let backoff = 100
+  for (let attempt = 1; attempt <= NUM_RETRIES; attempt++) {
+    try {
+      return await originalFetch.apply(globalThis, args)
+    } catch (error) {
+      let shouldRetry = false
+      // not ideal, but there is no error code for that
+      if (error.message === 'fetch failed' && attempt < NUM_RETRIES) {
+        // on this error we try again
+        shouldRetry = true
+      }
+
+      if (shouldRetry) {
+        // leave some trace in logs what's happening
+        console.error('[fetch-retry] fetch thrown, retrying...', {
+          args,
+          attempt,
+          errorMsg: error.message,
+        })
+
+        const currentBackoff = backoff
+        await new Promise((resolve) => {
+          setTimeout(resolve, currentBackoff)
+        })
+        backoff *= 2
+        continue
+      }
+
+      throw error
+    }
+  }
+}