Retry manifest file loading only in dev mode (#73900)

In #73891 we added another manifest to be loaded in `loadComponents`
(initially unconditionally). This uncovered a flakiness in prod mode
when attempting to load an optional manifest. The non-existent manifest
is attempted to be loaded three times with 100ms delay between attempts,
before giving up. For some reason the increased loading time leads to
more test flakiness.

To mitigate this, we're limiting the retry behaviour to the dev mode,
which matches the original intention when this was introduced in #45244.

Author: unstubbable

packages/next/src/build/utils.ts

@@ -1569,6 +1569,7 @@ export async function isPageStatic({
           distDir,
           page: originalAppPath || page,
           isAppPath: pageType === 'app',
+          isDev: false,
         })
       }
       const Comp = componentsResult.Component as NextComponentType | undefined
@@ -1802,6 +1803,7 @@ export async function hasCustomGetInitialProps({
     distDir,
     page: page,
     isAppPath: false,
+    isDev: false,
   })
   let mod = components.ComponentMod
 
@@ -1828,6 +1830,7 @@ export async function getDefinedNamedExports({
     distDir,
     page: page,
     isAppPath: false,
+    isDev: false,
   })
 
   return Object.keys(components.ComponentMod).filter((key) => {

packages/next/src/export/worker.ts

@@ -257,6 +257,7 @@ async function exportPageImpl(
       distDir,
       page,
       isAppPath: isAppDir,
+      isDev: false,
     })
 
     const renderOpts: WorkerRenderOpts = {

packages/next/src/server/dev/static-paths-worker.ts

@@ -71,6 +71,7 @@ export async function loadStaticPaths({
     // In `pages/`, the page is the same as the pathname.
     page: page || pathname,
     isAppPath,
+    isDev: true,
   })
 
   if (!components.getStaticPaths && !isAppPath) {

packages/next/src/server/load-components.ts

@@ -110,12 +110,13 @@ export async function evalManifestWithRetries<T extends object>(
 
 async function loadClientReferenceManifest(
   manifestPath: string,
-  entryName: string
+  entryName: string,
+  attempts?: number
 ) {
   try {
     const context = await evalManifestWithRetries<{
       __RSC_MANIFEST: { [key: string]: ClientReferenceManifest }
-    }>(manifestPath)
+    }>(manifestPath, attempts)
     return context.__RSC_MANIFEST[entryName]
   } catch (err) {
     return undefined
@@ -126,10 +127,12 @@ async function loadComponentsImpl<N = any>({
   distDir,
   page,
   isAppPath,
+  isDev,
 }: {
   distDir: string
   page: string
   isAppPath: boolean
+  isDev: boolean
 }): Promise<LoadComponentsReturnType<N>> {
   let DocumentMod = {}
   let AppMod = {}
@@ -144,16 +147,26 @@ async function loadComponentsImpl<N = any>({
   const hasClientManifest =
     isAppPath && (page.endsWith('/page') || page === UNDERSCORE_NOT_FOUND_ROUTE)
 
+  // In dev mode we retry loading a manifest file to handle a race condition
+  // that can occur while app and pages are compiling at the same time, and the
+  // build-manifest is still being written to disk while an app path is
+  // attempting to load.
+  const manifestLoadAttempts = isDev ? 3 : 1
+
   // Load the manifest files first
   const [
     buildManifest,
     reactLoadableManifest,
     clientReferenceManifest,
     serverActionsManifest,
   ] = await Promise.all([
-    loadManifestWithRetries<BuildManifest>(join(distDir, BUILD_MANIFEST)),
+    loadManifestWithRetries<BuildManifest>(
+      join(distDir, BUILD_MANIFEST),
+      manifestLoadAttempts
+    ),
     loadManifestWithRetries<ReactLoadableManifest>(
-      join(distDir, REACT_LOADABLE_MANIFEST)
+      join(distDir, REACT_LOADABLE_MANIFEST),
+      manifestLoadAttempts
     ),
     hasClientManifest
       ? loadClientReferenceManifest(
@@ -163,12 +176,14 @@ async function loadComponentsImpl<N = any>({
             'app',
             page.replace(/%5F/g, '_') + '_' + CLIENT_REFERENCE_MANIFEST + '.js'
           ),
-          page.replace(/%5F/g, '_')
+          page.replace(/%5F/g, '_'),
+          manifestLoadAttempts
         )
       : undefined,
     isAppPath
       ? loadManifestWithRetries<ActionManifest>(
-          join(distDir, 'server', SERVER_REFERENCE_MANIFEST + '.json')
+          join(distDir, 'server', SERVER_REFERENCE_MANIFEST + '.json'),
+          manifestLoadAttempts
         ).catch(() => null)
       : null,
   ])

packages/next/src/server/next-server.ts

@@ -179,11 +179,14 @@ export default class NextNodeServer extends BaseServer {
     req: IncomingMessage,
     res: ServerResponse
   ) => void
+  private isDev: boolean
 
   constructor(options: Options) {
     // Initialize super class
     super(options)
 
+    this.isDev = options.dev ?? false
+
     /**
      * This sets environment variable to be used at the time of SSR by head.tsx.
      * Using this from process.env allows targeting SSR by calling
@@ -220,11 +223,13 @@ export default class NextNodeServer extends BaseServer {
         distDir: this.distDir,
         page: '/_document',
         isAppPath: false,
+        isDev: this.isDev,
       }).catch(() => {})
       loadComponents({
         distDir: this.distDir,
         page: '/_app',
         isAppPath: false,
+        isDev: this.isDev,
       }).catch(() => {})
     }
 
@@ -281,11 +286,17 @@ export default class NextNodeServer extends BaseServer {
         distDir: this.distDir,
         page,
         isAppPath: false,
+        isDev: this.isDev,
       }).catch(() => {})
     }
 
     for (const page of Object.keys(appPathsManifest || {})) {
-      await loadComponents({ distDir: this.distDir, page, isAppPath: true })
+      await loadComponents({
+        distDir: this.distDir,
+        page,
+        isAppPath: true,
+        isDev: this.isDev,
+      })
         .then(async ({ ComponentMod }) => {
           const webpackRequire = ComponentMod.__next_app__.require
           if (webpackRequire?.m) {
@@ -758,6 +769,7 @@ export default class NextNodeServer extends BaseServer {
           distDir: this.distDir,
           page: pagePath,
           isAppPath,
+          isDev: this.isDev,
         })
 
         if (