fix duplicated noindex when server action is triggered (#76847)

### What

When client sends and Server Action request with `notFound()` inside,
next-server will serve the RSC payload in response but we were also
serving a fallback `robots=noindex` metadata for 404 requests. This will
be a conflict for users custom metadata. The fallback `robots=noindex`
for 404 response was for initially for pages rendering request.

Here we skip it for the dynamic RSC to avoid user's metadata being
de-prioritized due to the fallback `robots=noindex` taking precedence.

We also add a test that it won't break form action, where when JS is
disabled and notFound involked, the new page can still be served with
fallback noindex

Fixes NDX-953

Author: huozhi

packages/next/src/server/app-render/app-render.tsx

@@ -427,7 +427,8 @@ function NonIndex({ ctx }: { ctx: AppRenderContext }) {
   const isInvalidStatusCode =
     typeof ctx.res.statusCode === 'number' && ctx.res.statusCode > 400
 
-  if (is404Page || isInvalidStatusCode) {
+  // Only render noindex for page request, skip for server actions
+  if (!ctx.isAction && (is404Page || isInvalidStatusCode)) {
     return <meta name="robots" content="noindex" />
   }
   return null

test/e2e/app-dir/actions/app-action.test.ts

@@ -359,9 +359,15 @@ describe('app-dir action handling', () => {
 
     await browser.elementByCss('#nowhere').click()
 
+    // Until not-found page is resolved
     await retry(async () => {
       expect(await browser.elementByCss('h1').text()).toBe('my-not-found')
     })
+
+    // Should have default noindex meta tag
+    expect(
+      await browser.elementByCss('meta[name="robots"]').getAttribute('content')
+    ).toBe('noindex')
   })
 
   it('should support uploading files', async () => {

test/e2e/app-dir/metadata-navigation/app/server-action/not-found/action.tsx

@@ -0,0 +1,7 @@
+'use server'
+
+import { notFound } from 'next/navigation'
+
+export async function actionNotFound() {
+  return notFound()
+}

test/e2e/app-dir/metadata-navigation/app/server-action/not-found/page-client.tsx

@@ -0,0 +1,16 @@
+'use client'
+
+import { actionNotFound } from './action'
+
+export default function Page() {
+  return (
+    <button
+      id="trigger-not-found"
+      onClick={() => {
+        actionNotFound()
+      }}
+    >
+      trigger not found
+    </button>
+  )
+}

test/e2e/app-dir/metadata-navigation/app/server-action/not-found/page.tsx

@@ -0,0 +1,7 @@
+export { default } from './page-client'
+
+export async function generateMetadata() {
+  return {
+    robots: 'noindex, nofollow',
+  }
+}

test/e2e/app-dir/metadata-navigation/metadata-navigation.test.ts

@@ -5,6 +5,7 @@ import {
   getTitle,
   retry,
 } from 'next-test-utils'
+import { Request } from 'playwright'
 
 describe('app dir - metadata navigation', () => {
   const { next } = nextTestSetup({
@@ -108,4 +109,33 @@ describe('app dir - metadata navigation', () => {
       expect(await browser.elementByCss('title').text()).toBe('Home Layout')
     })
   })
+
+  describe('server action', () => {
+    it('should not render fallback noindex metadata if request is initiated from server action', async () => {
+      const browser = await next.browser('/server-action/not-found')
+      // collect server action requests
+      let isActionSent = false
+      browser.on('request', (req: Request) => {
+        if (
+          req.method() === 'POST' &&
+          req.url().endsWith('/server-action/not-found')
+        ) {
+          isActionSent = true
+        }
+      })
+
+      // trigger not-found action and wait until the server action is performed
+      await browser.elementByCss('#trigger-not-found').click()
+      await retry(async () => {
+        expect(isActionSent).toBe(true)
+      })
+
+      expect(await browser.elementsByCss('meta[name="robots"]')).toHaveLength(1)
+      expect(
+        await browser
+          .elementByCss('meta[name="robots"]')
+          .getAttribute('content')
+      ).toBe('noindex, nofollow')
+    })
+  })
 })