Add calling getStaticPaths in development before showing fallback (#10611)

* Add calling getStaticPaths in development before showing fallback

* Move staticPathsWorker to next-dev-server

* Make sure to clear require cache in worker process

* bump

* Remove staticPathsCache member

* Update numWorkers for staticPathsWorker

packages/next/build/utils.ts

@@ -497,6 +497,113 @@ export async function getPageSizeInKb(
   return [-1, -1]
 }
 
+export async function buildStaticPaths(
+  page: string,
+  unstable_getStaticPaths: Unstable_getStaticPaths
+): Promise<Array<string>> {
+  const prerenderPaths = new Set<string>()
+  const _routeRegex = getRouteRegex(page)
+  const _routeMatcher = getRouteMatcher(_routeRegex)
+
+  // Get the default list of allowed params.
+  const _validParamKeys = Object.keys(_routeMatcher(page))
+
+  const staticPathsResult = await unstable_getStaticPaths()
+
+  const expectedReturnVal =
+    `Expected: { paths: [] }\n` +
+    `See here for more info: https://err.sh/zeit/next.js/invalid-getstaticpaths-value`
+
+  if (
+    !staticPathsResult ||
+    typeof staticPathsResult !== 'object' ||
+    Array.isArray(staticPathsResult)
+  ) {
+    throw new Error(
+      `Invalid value returned from unstable_getStaticPaths in ${page}. Received ${typeof staticPathsResult} ${expectedReturnVal}`
+    )
+  }
+
+  const invalidStaticPathKeys = Object.keys(staticPathsResult).filter(
+    key => key !== 'paths'
+  )
+
+  if (invalidStaticPathKeys.length > 0) {
+    throw new Error(
+      `Extra keys returned from unstable_getStaticPaths in ${page} (${invalidStaticPathKeys.join(
+        ', '
+      )}) ${expectedReturnVal}`
+    )
+  }
+
+  const toPrerender = staticPathsResult.paths
+
+  if (!Array.isArray(toPrerender)) {
+    throw new Error(
+      `Invalid \`paths\` value returned from unstable_getStaticProps in ${page}.\n` +
+        `\`paths\` must be an array of strings or objects of shape { params: [key: string]: string }`
+    )
+  }
+
+  toPrerender.forEach(entry => {
+    // For a string-provided path, we must make sure it matches the dynamic
+    // route.
+    if (typeof entry === 'string') {
+      const result = _routeMatcher(entry)
+      if (!result) {
+        throw new Error(
+          `The provided path \`${entry}\` does not match the page: \`${page}\`.`
+        )
+      }
+
+      prerenderPaths?.add(entry)
+    }
+    // For the object-provided path, we must make sure it specifies all
+    // required keys.
+    else {
+      const invalidKeys = Object.keys(entry).filter(key => key !== 'params')
+      if (invalidKeys.length) {
+        throw new Error(
+          `Additional keys were returned from \`unstable_getStaticPaths\` in page "${page}". ` +
+            `URL Parameters intended for this dynamic route must be nested under the \`params\` key, i.e.:` +
+            `\n\n\treturn { params: { ${_validParamKeys
+              .map(k => `${k}: ...`)
+              .join(', ')} } }` +
+            `\n\nKeys that need to be moved: ${invalidKeys.join(', ')}.\n`
+        )
+      }
+
+      const { params = {} } = entry
+      let builtPage = page
+      _validParamKeys.forEach(validParamKey => {
+        const { repeat } = _routeRegex.groups[validParamKey]
+        const paramValue = params[validParamKey]
+        if (
+          (repeat && !Array.isArray(paramValue)) ||
+          (!repeat && typeof paramValue !== 'string')
+        ) {
+          throw new Error(
+            `A required parameter (${validParamKey}) was not provided as ${
+              repeat ? 'an array' : 'a string'
+            } in unstable_getStaticPaths for ${page}`
+          )
+        }
+
+        builtPage = builtPage.replace(
+          `[${repeat ? '...' : ''}${validParamKey}]`,
+          repeat
+            ? (paramValue as string[]).map(encodeURIComponent).join('/')
+            : encodeURIComponent(paramValue as string)
+        )
+      })
+
+      prerenderPaths?.add(builtPage)
+    }
+  })
+
+  return [...prerenderPaths]
+}
+
 export async function isPageStatic(
   page: string,
   serverBundle: string,
@@ -559,115 +666,19 @@ export async function isPageStatic(
       )
     }
 
-    let prerenderPaths: Set<string> | undefined
+    let prerenderRoutes: Array<string> | undefined
     if (hasStaticProps && hasStaticPaths) {
-      prerenderPaths = new Set()
-
-      const _routeRegex = getRouteRegex(page)
-      const _routeMatcher = getRouteMatcher(_routeRegex)
-
-      // Get the default list of allowed params.
-      const _validParamKeys = Object.keys(_routeMatcher(page))
-
-      const staticPathsResult = await (mod.unstable_getStaticPaths as Unstable_getStaticPaths)()
-
-      const expectedReturnVal =
-        `Expected: { paths: [] }\n` +
-        `See here for more info: https://err.sh/zeit/next.js/invalid-getstaticpaths-value`
-
-      if (
-        !staticPathsResult ||
-        typeof staticPathsResult !== 'object' ||
-        Array.isArray(staticPathsResult)
-      ) {
-        throw new Error(
-          `Invalid value returned from unstable_getStaticPaths in ${page}. Received ${typeof staticPathsResult} ${expectedReturnVal}`
-        )
-      }
-
-      const invalidStaticPathKeys = Object.keys(staticPathsResult).filter(
-        key => key !== 'paths'
+      prerenderRoutes = await buildStaticPaths(
+        page,
+        mod.unstable_getStaticPaths
       )
-
-      if (invalidStaticPathKeys.length > 0) {
-        throw new Error(
-          `Extra keys returned from unstable_getStaticPaths in ${page} (${invalidStaticPathKeys.join(
-            ', '
-          )}) ${expectedReturnVal}`
-        )
-      }
-
-      const toPrerender = staticPathsResult.paths
-
-      if (!Array.isArray(toPrerender)) {
-        throw new Error(
-          `Invalid \`paths\` value returned from unstable_getStaticProps in ${page}.\n` +
-            `\`paths\` must be an array of strings or objects of shape { params: [key: string]: string }`
-        )
-      }
-
-      toPrerender.forEach(entry => {
-        // For a string-provided path, we must make sure it matches the dynamic
-        // route.
-        if (typeof entry === 'string') {
-          const result = _routeMatcher(entry)
-          if (!result) {
-            throw new Error(
-              `The provided path \`${entry}\` does not match the page: \`${page}\`.`
-            )
-          }
-
-          prerenderPaths?.add(entry)
-        }
-        // For the object-provided path, we must make sure it specifies all
-        // required keys.
-        else {
-          const invalidKeys = Object.keys(entry).filter(key => key !== 'params')
-          if (invalidKeys.length) {
-            throw new Error(
-              `Additional keys were returned from \`unstable_getStaticPaths\` in page "${page}". ` +
-                `URL Parameters intended for this dynamic route must be nested under the \`params\` key, i.e.:` +
-                `\n\n\treturn { params: { ${_validParamKeys
-                  .map(k => `${k}: ...`)
-                  .join(', ')} } }` +
-                `\n\nKeys that need to be moved: ${invalidKeys.join(', ')}.\n`
-            )
-          }
-
-          const { params = {} } = entry
-          let builtPage = page
-          _validParamKeys.forEach(validParamKey => {
-            const { repeat } = _routeRegex.groups[validParamKey]
-            const paramValue = params[validParamKey]
-            if (
-              (repeat && !Array.isArray(paramValue)) ||
-              (!repeat && typeof paramValue !== 'string')
-            ) {
-              throw new Error(
-                `A required parameter (${validParamKey}) was not provided as ${
-                  repeat ? 'an array' : 'a string'
-                } in unstable_getStaticPaths for ${page}`
-              )
-            }
-
-            builtPage = builtPage.replace(
-              `[${repeat ? '...' : ''}${validParamKey}]`,
-              repeat
-                ? (paramValue as string[]).map(encodeURIComponent).join('/')
-                : encodeURIComponent(paramValue as string)
-            )
-          })
-
-          prerenderPaths?.add(builtPage)
-        }
-      })
     }
 
     const config = mod.config || {}
     return {
       isStatic: !hasStaticProps && !hasGetInitialProps && !hasServerProps,
       isHybridAmp: config.amp === 'hybrid',
-      prerenderRoutes: prerenderPaths && [...prerenderPaths],
+      prerenderRoutes,
       hasStaticProps,
       hasServerProps,
     }

packages/next/next-server/server/next-server.ts

@@ -124,6 +124,9 @@ export default class Server {
     redirects: Redirect[]
     headers: Header[]
   }
+  protected staticPathsWorker?: import('jest-worker').default & {
+    loadStaticPaths: typeof import('../../server/static-paths-worker').loadStaticPaths
+  }
 
   public constructor({
     dir = '.',
@@ -884,6 +887,7 @@ export default class Server {
       typeof (components.Component as any).renderReqToHTML === 'function'
     const isSSG = !!components.unstable_getStaticProps
     const isServerProps = !!components.unstable_getServerProps
+    const hasStaticPaths = !!components.unstable_getStaticPaths
 
     // Toggle whether or not this is a Data request
     const isDataReq = query._nextDataReq
@@ -950,9 +954,10 @@ export default class Server {
     const isPreviewMode = previewData !== false
 
     // Compute the SPR cache key
+    const urlPathname = parseUrl(req.url || '').pathname!
     const ssgCacheKey = isPreviewMode
       ? `__` + nanoid() // Preview mode uses a throw away key to not coalesce preview invokes
-      : parseUrl(req.url || '').pathname!
+      : urlPathname
 
     // Complete the response with cached data if its present
     const cachedData = isPreviewMode
@@ -1020,6 +1025,30 @@ export default class Server {
     const isProduction = !this.renderOpts.dev
     const isDynamicPathname = isDynamicRoute(pathname)
     const didRespond = isResSent(res)
+
+    // we lazy load the staticPaths to prevent the user
+    // from waiting on them for the page to load in dev mode
+    let staticPaths: string[] | undefined
+
+    if (!isProduction && hasStaticPaths) {
+      const __getStaticPaths = async () => {
+        const paths = await this.staticPathsWorker!.loadStaticPaths(
+          this.distDir,
+          this.buildId,
+          pathname,
+          !this.renderOpts.dev && this._isLikeServerless
+        )
+        return paths
+      }
+
+      staticPaths = (
+        await withCoalescedInvoke(__getStaticPaths)(
+          `staticPaths-${pathname}`,
+          []
+        )
+      ).value
+    }
+
     // const isForcedBlocking =
     //   req.headers['X-Prerender-Bypass-Mode'] !== 'Blocking'
 
@@ -1030,20 +1059,20 @@ export default class Server {
     //
     // * Preview mode toggles all pages to be resolved in a blocking manner.
     //
-    // * Non-dynamic pages should block (though this is an be an impossible
+    // * Non-dynamic pages should block (though this is an impossible
     //   case in production).
     //
-    // * Dynamic pages should return their skeleton, then finish the data
-    //   request on the client-side.
+    // * Dynamic pages should return their skeleton if not defined in
+    //   getStaticPaths, then finish the data request on the client-side.
     //
     if (
       !didRespond &&
       !isDataReq &&
       !isPreviewMode &&
       isDynamicPathname &&
-      // TODO: development should trigger fallback when the path is not in
-      // `getStaticPaths`, for now, let's assume it is.
-      isProduction
+      // Development should trigger fallback when the path is not in
+      // `getStaticPaths`
+      (isProduction || !staticPaths || !staticPaths.includes(urlPathname))
     ) {
       let html: string
 

packages/next/server/next-dev-server.ts

@@ -30,6 +30,7 @@ import { Telemetry } from '../telemetry/storage'
 import ErrorDebug from './error-debug'
 import HotReloader from './hot-reloader'
 import { findPageFile } from './lib/find-page-file'
+import Worker from 'jest-worker'
 
 if (typeof React.Suspense === 'undefined') {
   throw new Error(
@@ -79,6 +80,18 @@ export default class DevServer extends Server {
     }
     this.isCustomServer = !options.isNextDevCommand
     this.pagesDir = findPagesDir(this.dir)
+    this.staticPathsWorker = new Worker(
+      require.resolve('./static-paths-worker'),
+      {
+        maxRetries: 0,
+        numWorkers: this.nextConfig.experimental.cpus,
+      }
+    ) as Worker & {
+      loadStaticPaths: typeof import('./static-paths-worker').loadStaticPaths
+    }
+
+    this.staticPathsWorker.getStdout().pipe(process.stdout)
+    this.staticPathsWorker.getStderr().pipe(process.stderr)
   }
 
   protected currentPhase() {

packages/next/server/static-paths-worker.ts

+import { join } from 'path'
+import { buildStaticPaths } from '../build/utils'
+import { getPagePath } from '../next-server/server/require'
+import { loadComponents } from '../next-server/server/load-components'
+import { PAGES_MANIFEST, SERVER_DIRECTORY } from '../next-server/lib/constants'
+
+// we call getStaticPaths in a separate process to ensure
+// side-effects aren't relied on in dev that will break
+// during a production build
+export async function loadStaticPaths(
+  distDir: string,
+  buildId: string,
+  pathname: string,
+  serverless: boolean
+) {
+  // we need to clear any modules manually here since the
+  // require-cache-hot-loader doesn't affect require cache here
+  // since we're in a separate process
+  delete require.cache[join(distDir, SERVER_DIRECTORY, PAGES_MANIFEST)]
+
+  const pagePath = await getPagePath(pathname, distDir, serverless, true)
+  delete require.cache[pagePath]
+
+  const components = await loadComponents(
+    distDir,
+    buildId,
+    pathname,
+    serverless
+  )
+
+  if (!components.unstable_getStaticPaths) {
+    // we shouldn't get to this point since the worker should
+    // only be called for SSG pages with getStaticPaths
+    throw new Error(
+      `Invariant: failed to load page with unstable_getStaticPaths for ${pathname}`
+    )
+  }
+
+  return buildStaticPaths(pathname, components.unstable_getStaticPaths)
+}

test/integration/prerender/test/index.test.js

@@ -381,61 +381,41 @@ const runTests = (dev = false, looseMode = false) => {
   })
 
   it('should support lazy catchall route', async () => {
-    // Dev doesn't support fallback yet
-    if (dev) {
-      const html = await renderViaHTTP(appPort, '/catchall/notreturnedinpaths')
-      const $ = cheerio.load(html)
-      expect($('#catchall').text()).toMatch(/Hi.*?notreturnedinpaths/)
-    }
-    // Production will render fallback for a "lazy" route
-    else {
-      const html = await renderViaHTTP(appPort, '/catchall/notreturnedinpaths')
-      const $ = cheerio.load(html)
-      expect($('#catchall').text()).toBe('fallback')
+    const html = await renderViaHTTP(appPort, '/catchall/notreturnedinpaths')
+    const $ = cheerio.load(html)
+    expect($('#catchall').text()).toBe('fallback')
 
-      // hydration
-      const browser = await webdriver(appPort, '/catchall/delayby3s')
+    // hydration
+    const browser = await webdriver(appPort, '/catchall/delayby3s')
 
-      const text1 = await browser.elementByCss('#catchall').text()
-      expect(text1).toBe('fallback')
+    const text1 = await browser.elementByCss('#catchall').text()
+    expect(text1).toBe('fallback')
 
-      await check(
-        () => browser.elementByCss('#catchall').text(),
-        /Hi.*?delayby3s/
-      )
-    }
+    await check(
+      () => browser.elementByCss('#catchall').text(),
+      /Hi.*?delayby3s/
+    )
   })
 
   it('should support nested lazy catchall route', async () => {
-    // Dev doesn't support fallback yet
-    if (dev) {
-      const html = await renderViaHTTP(
-        appPort,
-        '/catchall/notreturnedinpaths/nested'
-      )
-      const $ = cheerio.load(html)
-      expect($('#catchall').text()).toMatch(/Hi.*?notreturnedinpaths nested/)
-    }
-    // Production will render fallback for a "lazy" route
-    else {
-      const html = await renderViaHTTP(
-        appPort,
-        '/catchall/notreturnedinpaths/nested'
-      )
-      const $ = cheerio.load(html)
-      expect($('#catchall').text()).toBe('fallback')
+    // We will render fallback for a "lazy" route
+    const html = await renderViaHTTP(
+      appPort,
+      '/catchall/notreturnedinpaths/nested'
+    )
+    const $ = cheerio.load(html)
+    expect($('#catchall').text()).toBe('fallback')
 
-      // hydration
-      const browser = await webdriver(appPort, '/catchall/delayby3s/nested')
+    // hydration
+    const browser = await webdriver(appPort, '/catchall/delayby3s/nested')
 
-      const text1 = await browser.elementByCss('#catchall').text()
-      expect(text1).toBe('fallback')
+    const text1 = await browser.elementByCss('#catchall').text()
+    expect(text1).toBe('fallback')
 
-      await check(
-        () => browser.elementByCss('#catchall').text(),
-        /Hi.*?delayby3s nested/
-      )
-    }
+    await check(
+      () => browser.elementByCss('#catchall').text(),
+      /Hi.*?delayby3s nested/
+    )
   })
 
   if (dev) {
@@ -448,6 +428,28 @@ const runTests = (dev = false, looseMode = false) => {
     //   )
     // })
 
+    it('should always show fallback for page not in getStaticPaths', async () => {
+      const html = await renderViaHTTP(appPort, '/blog/post-321')
+      const $ = cheerio.load(html)
+      expect(JSON.parse($('#__NEXT_DATA__').text()).isFallback).toBe(true)
+
+      // make another request to ensure it still is
+      const html2 = await renderViaHTTP(appPort, '/blog/post-321')
+      const $2 = cheerio.load(html2)
+      expect(JSON.parse($2('#__NEXT_DATA__').text()).isFallback).toBe(true)
+    })
+
+    it('should not show fallback for page in getStaticPaths', async () => {
+      const html = await renderViaHTTP(appPort, '/blog/post-1')
+      const $ = cheerio.load(html)
+      expect(JSON.parse($('#__NEXT_DATA__').text()).isFallback).toBe(false)
+
+      // make another request to ensure it's still not
+      const html2 = await renderViaHTTP(appPort, '/blog/post-1')
+      const $2 = cheerio.load(html2)
+      expect(JSON.parse($2('#__NEXT_DATA__').text()).isFallback).toBe(false)
+    })
+
     it('should log error in console and browser in dev mode', async () => {
       const origContent = await fs.readFile(indexPage, 'utf8')