Enable handling for revalidate and notFound (#19165)

This allows SSG pages that return `notFound` to be revalidated. 

Closes: https://github.com/vercel/next.js/issues/18651

packages/next/next-server/server/next-server.ts

@@ -65,7 +65,7 @@ import Router, {
 import prepareDestination, {
   compileNonPath,
 } from '../lib/router/utils/prepare-destination'
-import { sendPayload } from './send-payload'
+import { sendPayload, setRevalidateHeaders } from './send-payload'
 import { serveStatic } from './serve-static'
 import { IncrementalCache } from './incremental-cache'
 import { execOnce } from '../lib/utils'
@@ -1272,9 +1272,6 @@ export default class Server {
       ? this.nextConfig.i18n?.defaultLocale
       : (query.__nextDefaultLocale as string)
 
-    delete query.__nextLocale
-    delete query.__nextDefaultLocale
-
     const { i18n } = this.nextConfig
     const locales = i18n.locales as string[]
 
@@ -1361,18 +1358,31 @@ export default class Server {
       : undefined
 
     if (cachedData) {
-      if (cachedData.isNotFound) {
-        // we don't currently revalidate when notFound is returned
-        // so trigger rendering 404 here
-        throw new NoFallbackError()
-      }
-
       const data = isDataReq
         ? JSON.stringify(cachedData.pageData)
         : cachedData.html
 
+      const revalidateOptions = !this.renderOpts.dev
+        ? {
+            private: isPreviewMode,
+            stateful: false, // GSP response
+            revalidate:
+              cachedData.curRevalidate !== undefined
+                ? cachedData.curRevalidate
+                : /* default to minimum revalidate (this should be an invariant) */ 1,
+          }
+        : undefined
+
       if (!isDataReq && cachedData.pageData?.pageProps?.__N_REDIRECT) {
         await handleRedirect(cachedData.pageData)
+      } else if (cachedData.isNotFound) {
+        if (revalidateOptions) {
+          setRevalidateHeaders(res, revalidateOptions)
+        }
+        await this.render404(req, res, {
+          pathname,
+          query,
+        } as UrlWithParsedQuery)
       } else {
         sendPayload(
           req,
@@ -1383,16 +1393,7 @@ export default class Server {
             generateEtags: this.renderOpts.generateEtags,
             poweredByHeader: this.renderOpts.poweredByHeader,
           },
-          !this.renderOpts.dev
-            ? {
-                private: isPreviewMode,
-                stateful: false, // GSP response
-                revalidate:
-                  cachedData.curRevalidate !== undefined
-                    ? cachedData.curRevalidate
-                    : /* default to minimum revalidate (this should be an invariant) */ 1,
-              }
-            : undefined
+          revalidateOptions
         )
       }
 
@@ -1575,6 +1576,15 @@ export default class Server {
     } = await doRender()
     let resHtml = html
 
+    const revalidateOptions =
+      !this.renderOpts.dev || (isServerProps && !isDataReq)
+        ? {
+            private: isPreviewMode,
+            stateful: !isSSG,
+            revalidate: sprRevalidate,
+          }
+        : undefined
+
     if (
       !isResSent(res) &&
       !isNotFound &&
@@ -1592,13 +1602,7 @@ export default class Server {
             generateEtags: this.renderOpts.generateEtags,
             poweredByHeader: this.renderOpts.poweredByHeader,
           },
-          !this.renderOpts.dev || (isServerProps && !isDataReq)
-            ? {
-                private: isPreviewMode,
-                stateful: !isSSG,
-                revalidate: sprRevalidate,
-              }
-            : undefined
+          revalidateOptions
         )
       }
       resHtml = null
@@ -1613,8 +1617,16 @@ export default class Server {
       )
     }
 
-    if (isNotFound) {
-      throw new NoFallbackError()
+    if (!isResSent(res) && isNotFound) {
+      if (revalidateOptions) {
+        setRevalidateHeaders(res, revalidateOptions)
+      }
+      await this.render404(
+        req,
+        res,
+        { pathname, query } as UrlWithParsedQuery,
+        !!revalidateOptions
+      )
     }
     return resHtml
   }
@@ -1691,12 +1703,15 @@ export default class Server {
     req: IncomingMessage,
     res: ServerResponse,
     pathname: string,
-    query: ParsedUrlQuery = {}
+    query: ParsedUrlQuery = {},
+    setHeaders = true
   ): Promise<void> {
-    res.setHeader(
-      'Cache-Control',
-      'no-cache, no-store, max-age=0, must-revalidate'
-    )
+    if (setHeaders) {
+      res.setHeader(
+        'Cache-Control',
+        'no-cache, no-store, max-age=0, must-revalidate'
+      )
+    }
     const html = await this.renderErrorToHTML(err, req, res, pathname, query)
     if (html === null) {
       return
@@ -1774,12 +1789,13 @@ export default class Server {
   public async render404(
     req: IncomingMessage,
     res: ServerResponse,
-    parsedUrl?: UrlWithParsedQuery
+    parsedUrl?: UrlWithParsedQuery,
+    setHeaders = true
   ): Promise<void> {
     const url: any = req.url
     const { pathname, query } = parsedUrl ? parsedUrl : parseUrl(url, true)
     res.statusCode = 404
-    return this.renderError(null, req, res, pathname!, query)
+    return this.renderError(null, req, res, pathname!, query, setHeaders)
   }
 
   public async serveStatic(

packages/next/next-server/server/render.tsx

@@ -353,6 +353,9 @@ export async function renderToHTML(
     ? renderOpts.devOnlyCacheBusterQueryString || `?ts=${Date.now()}`
     : ''
 
+  // don't modify original query object
+  query = Object.assign({}, query)
+
   const {
     err,
     dev = false,
@@ -654,8 +657,6 @@ export async function renderToHTML(
         }
 
         ;(renderOpts as any).isNotFound = true
-        ;(renderOpts as any).revalidate = false
-        return null
       }
 
       if (
@@ -684,6 +685,7 @@ export async function renderToHTML(
 
       if (
         (dev || isBuildTimeSSG) &&
+        !(renderOpts as any).isNotFound &&
         !isSerializableProps(pathname, 'getStaticProps', (data as any).props)
       ) {
         // this fn should throw an error instead of ever returning `false`
@@ -723,6 +725,11 @@ export async function renderToHTML(
         ;(data as any).revalidate = false
       }
 
+      // this must come after revalidate is attached
+      if ((renderOpts as any).isNotFound) {
+        return null
+      }
+
       props.pageProps = Object.assign(
         {},
         props.pageProps,

packages/next/next-server/server/send-payload.ts

@@ -3,6 +3,38 @@ import { isResSent } from '../lib/utils'
 import generateETag from 'etag'
 import fresh from 'next/dist/compiled/fresh'
 
+type PayloadOptions =
+  | { private: true }
+  | { private: boolean; stateful: true }
+  | { private: boolean; stateful: false; revalidate: number | false }
+
+export function setRevalidateHeaders(
+  res: ServerResponse,
+  options: PayloadOptions
+) {
+  if (options.private || options.stateful) {
+    if (options.private || !res.hasHeader('Cache-Control')) {
+      res.setHeader(
+        'Cache-Control',
+        `private, no-cache, no-store, max-age=0, must-revalidate`
+      )
+    }
+  } else if (typeof options.revalidate === 'number') {
+    if (options.revalidate < 1) {
+      throw new Error(
+        `invariant: invalid Cache-Control duration provided: ${options.revalidate} < 1`
+      )
+    }
+
+    res.setHeader(
+      'Cache-Control',
+      `s-maxage=${options.revalidate}, stale-while-revalidate`
+    )
+  } else if (options.revalidate === false) {
+    res.setHeader('Cache-Control', `s-maxage=31536000, stale-while-revalidate`)
+  }
+}
+
 export function sendPayload(
   req: IncomingMessage,
   res: ServerResponse,
@@ -12,10 +44,7 @@ export function sendPayload(
     generateEtags,
     poweredByHeader,
   }: { generateEtags: boolean; poweredByHeader: boolean },
-  options?:
-    | { private: true }
-    | { private: boolean; stateful: true }
-    | { private: boolean; stateful: false; revalidate: number | false }
+  options?: PayloadOptions
 ): void {
   if (isResSent(res)) {
     return
@@ -38,30 +67,7 @@ export function sendPayload(
   }
   res.setHeader('Content-Length', Buffer.byteLength(payload))
   if (options != null) {
-    if (options.private || options.stateful) {
-      if (options.private || !res.hasHeader('Cache-Control')) {
-        res.setHeader(
-          'Cache-Control',
-          `private, no-cache, no-store, max-age=0, must-revalidate`
-        )
-      }
-    } else if (typeof options.revalidate === 'number') {
-      if (options.revalidate < 1) {
-        throw new Error(
-          `invariant: invalid Cache-Control duration provided: ${options.revalidate} < 1`
-        )
-      }
-
-      res.setHeader(
-        'Cache-Control',
-        `s-maxage=${options.revalidate}, stale-while-revalidate`
-      )
-    } else if (options.revalidate === false) {
-      res.setHeader(
-        'Cache-Control',
-        `s-maxage=31536000, stale-while-revalidate`
-      )
-    }
+    setRevalidateHeaders(res, options)
   }
   res.end(req.method === 'HEAD' ? null : payload)
 }

test/integration/not-found-revalidate/pages/404.js

+export default function Page(props) {
+  return (
+    <>
+      <p id="not-found">404 page</p>
+      <p id="props">{JSON.stringify(props)}</p>
+    </>
+  )
+}
+
+export const getStaticProps = () => {
+  return {
+    props: {
+      notFound: true,
+      random: Math.random(),
+    },
+    revalidate: 1,
+  }
+}

test/integration/not-found-revalidate/pages/fallback-blocking/[slug].js

+import path from 'path'
+import fs from 'fs-extra'
+
+export default function Page(props) {
+  return (
+    <>
+      <p id="fallback-blocking">fallback blocking page</p>
+      <p id="props">{JSON.stringify(props)}</p>
+    </>
+  )
+}
+
+export const getStaticProps = async ({ params }) => {
+  const { slug } = params
+
+  if (!slug) {
+    throw new Error(`missing slug value for /fallback-true/[slug]`)
+  }
+
+  const dir = path.join(process.cwd(), '.tmp/fallback-blocking', slug)
+
+  if (await fs.exists(dir)) {
+    return {
+      props: {
+        params,
+        found: true,
+        hello: 'world',
+        random: Math.random(),
+      },
+      revalidate: 1,
+    }
+  }
+
+  await fs.ensureDir(dir)
+
+  return {
+    notFound: true,
+    revalidate: 1,
+  }
+}
+
+export const getStaticPaths = async () => {
+  await fs.remove(path.join(process.cwd(), '.tmp/fallback-blocking'))
+
+  return {
+    paths: [],
+    fallback: 'blocking',
+  }
+}

test/integration/not-found-revalidate/pages/fallback-true/[slug].js

+import path from 'path'
+import fs from 'fs-extra'
+import { useRouter } from 'next/router'
+
+export default function Page(props) {
+  if (useRouter().isFallback) return 'Loading...'
+
+  return (
+    <>
+      <p id="fallback-true">fallback true page</p>
+      <p id="props">{JSON.stringify(props)}</p>
+    </>
+  )
+}
+
+export const getStaticProps = async ({ params }) => {
+  const { slug } = params
+
+  if (!slug) {
+    throw new Error(`missing slug value for /fallback-true/[slug]`)
+  }
+
+  const dir = path.join(process.cwd(), '.tmp/fallback-true', slug)
+
+  if (await fs.exists(dir)) {
+    return {
+      props: {
+        params,
+        found: true,
+        hello: 'world',
+        random: Math.random(),
+      },
+      revalidate: 1,
+    }
+  }
+
+  await fs.ensureDir(dir)
+
+  return {
+    notFound: true,
+    revalidate: 1,
+  }
+}
+
+export const getStaticPaths = async () => {
+  await fs.remove(path.join(process.cwd(), '.tmp/fallback-true'))
+
+  return {
+    paths: [],
+    fallback: true,
+  }
+}

test/integration/not-found-revalidate/test/index.test.js

+/* eslint-env jest */
+import fs from 'fs-extra'
+import { join } from 'path'
+import cheerio from 'cheerio'
+import webdriver from 'next-webdriver'
+import {
+  nextBuild,
+  nextStart,
+  findPort,
+  killApp,
+  fetchViaHTTP,
+  waitFor,
+} from 'next-test-utils'
+
+jest.setTimeout(1000 * 60 * 2)
+const appDir = join(__dirname, '..')
+let app
+let appPort
+
+const runTests = () => {
+  it('should revalidate after notFound is returned for fallback: blocking', async () => {
+    let res = await fetchViaHTTP(appPort, '/fallback-blocking/hello')
+    let $ = cheerio.load(await res.text())
+
+    expect(res.headers.get('cache-control')).toBe(
+      's-maxage=1, stale-while-revalidate'
+    )
+    expect(res.status).toBe(404)
+    expect(JSON.parse($('#props').text()).notFound).toBe(true)
+
+    await waitFor(1000)
+    res = await fetchViaHTTP(appPort, '/fallback-blocking/hello')
+    $ = cheerio.load(await res.text())
+
+    expect(res.headers.get('cache-control')).toBe(
+      's-maxage=1, stale-while-revalidate'
+    )
+    expect(res.status).toBe(404)
+    expect(JSON.parse($('#props').text()).notFound).toBe(true)
+
+    await waitFor(1000)
+    res = await fetchViaHTTP(appPort, '/fallback-blocking/hello')
+    $ = cheerio.load(await res.text())
+
+    const props = JSON.parse($('#props').text())
+    expect(res.headers.get('cache-control')).toBe(
+      's-maxage=1, stale-while-revalidate'
+    )
+    expect(res.status).toBe(200)
+    expect(props.found).toBe(true)
+    expect(props.params).toEqual({ slug: 'hello' })
+    expect(isNaN(props.random)).toBe(false)
+
+    await waitFor(1000)
+    res = await fetchViaHTTP(appPort, '/fallback-blocking/hello')
+    $ = cheerio.load(await res.text())
+
+    const props2 = JSON.parse($('#props').text())
+    expect(res.headers.get('cache-control')).toBe(
+      's-maxage=1, stale-while-revalidate'
+    )
+    expect(res.status).toBe(200)
+    expect(props2.found).toBe(true)
+    expect(props2.params).toEqual({ slug: 'hello' })
+    expect(isNaN(props2.random)).toBe(false)
+    expect(props2.random).not.toBe(props.random)
+  })
+
+  it('should revalidate after notFound is returned for fallback: true', async () => {
+    const browser = await webdriver(appPort, '/fallback-true/world')
+
+    await browser.waitForElementByCss('#not-found')
+
+    await waitFor(1000)
+    let res = await fetchViaHTTP(appPort, '/fallback-true/world')
+    let $ = cheerio.load(await res.text())
+
+    expect(res.headers.get('cache-control')).toBe(
+      's-maxage=1, stale-while-revalidate'
+    )
+    expect(res.status).toBe(404)
+    expect(JSON.parse($('#props').text()).notFound).toBe(true)
+
+    await waitFor(1000)
+    res = await fetchViaHTTP(appPort, '/fallback-true/world')
+    $ = cheerio.load(await res.text())
+
+    const props = JSON.parse($('#props').text())
+    expect(res.headers.get('cache-control')).toBe(
+      's-maxage=1, stale-while-revalidate'
+    )
+    expect(res.status).toBe(200)
+    expect(props.found).toBe(true)
+    expect(props.params).toEqual({ slug: 'world' })
+    expect(isNaN(props.random)).toBe(false)
+
+    await waitFor(1000)
+    res = await fetchViaHTTP(appPort, '/fallback-true/world')
+    $ = cheerio.load(await res.text())
+
+    const props2 = JSON.parse($('#props').text())
+    expect(res.headers.get('cache-control')).toBe(
+      's-maxage=1, stale-while-revalidate'
+    )
+    expect(res.status).toBe(200)
+    expect(props2.found).toBe(true)
+    expect(props2.params).toEqual({ slug: 'world' })
+    expect(isNaN(props2.random)).toBe(false)
+    expect(props2.random).not.toBe(props.random)
+  })
+}
+
+describe('SSG notFound revalidate', () => {
+  describe('production mode', () => {
+    beforeAll(async () => {
+      await fs.remove(join(appDir, '.next'))
+      await nextBuild(appDir)
+      appPort = await findPort()
+      app = await nextStart(appDir, appPort)
+    })
+    afterAll(() => killApp(app))
+
+    runTests()
+  })
+})