Add calling getStaticPaths in development before showing fallback (#10611)

* Add calling getStaticPaths in development before showing fallback

* Move staticPathsWorker to next-dev-server

* Make sure to clear require cache in worker process

* bump

* Remove staticPathsCache member

* Update numWorkers for staticPathsWorker

packages/next/build/utils.ts

@@ -497,79 +497,18 @@ export async function getPageSizeInKb(
   return [-1, -1]
 }
 
-export async function isPageStatic(
+export async function buildStaticPaths(
   page: string,
-  serverBundle: string,
-  runtimeEnvConfig: any
-): Promise<{
-  isStatic?: boolean
-  isHybridAmp?: boolean
-  hasServerProps?: boolean
-  hasStaticProps?: boolean
-  prerenderRoutes?: string[] | undefined
-}> {
-  try {
-    require('../next-server/lib/runtime-config').setConfig(runtimeEnvConfig)
-    const mod = require(serverBundle)
-    const Comp = mod.default || mod
-
-    if (!Comp || !isValidElementType(Comp) || typeof Comp === 'string') {
-      throw new Error('INVALID_DEFAULT_EXPORT')
-    }
-
-    const hasGetInitialProps = !!(Comp as any).getInitialProps
-    const hasStaticProps = !!mod.unstable_getStaticProps
-    const hasStaticPaths = !!mod.unstable_getStaticPaths
-    const hasServerProps = !!mod.unstable_getServerProps
-    const hasLegacyStaticParams = !!mod.unstable_getStaticParams
-
-    if (hasLegacyStaticParams) {
-      throw new Error(
-        `unstable_getStaticParams was replaced with unstable_getStaticPaths. Please update your code.`
-      )
-    }
-
-    // A page cannot be prerendered _and_ define a data requirement. That's
-    // contradictory!
-    if (hasGetInitialProps && hasStaticProps) {
-      throw new Error(SSG_GET_INITIAL_PROPS_CONFLICT)
-    }
-
-    if (hasGetInitialProps && hasServerProps) {
-      throw new Error(SERVER_PROPS_GET_INIT_PROPS_CONFLICT)
-    }
-
-    if (hasStaticProps && hasServerProps) {
-      throw new Error(SERVER_PROPS_SSG_CONFLICT)
-    }
-
-    const pageIsDynamic = isDynamicRoute(page)
-    // A page cannot have static parameters if it is not a dynamic page.
-    if (hasStaticProps && hasStaticPaths && !pageIsDynamic) {
-      throw new Error(
-        `unstable_getStaticPaths can only be used with dynamic pages, not '${page}'.` +
-          `\nLearn more: https://nextjs.org/docs#dynamic-routing`
-      )
-    }
-
-    if (hasStaticProps && pageIsDynamic && !hasStaticPaths) {
-      throw new Error(
-        `unstable_getStaticPaths is required for dynamic SSG pages and is missing for '${page}'.` +
-          `\nRead more: https://err.sh/next.js/invalid-getstaticpaths-value`
-      )
-    }
-
-    let prerenderPaths: Set<string> | undefined
-    if (hasStaticProps && hasStaticPaths) {
-      prerenderPaths = new Set()
-
+  unstable_getStaticPaths: Unstable_getStaticPaths
+): Promise<Array<string>> {
+  const prerenderPaths = new Set<string>()
   const _routeRegex = getRouteRegex(page)
   const _routeMatcher = getRouteMatcher(_routeRegex)
 
   // Get the default list of allowed params.
   const _validParamKeys = Object.keys(_routeMatcher(page))
 
-      const staticPathsResult = await (mod.unstable_getStaticPaths as Unstable_getStaticPaths)()
+  const staticPathsResult = await unstable_getStaticPaths()
 
   const expectedReturnVal =
     `Expected: { paths: [] }\n` +
@@ -661,13 +600,85 @@ export async function isPageStatic(
       prerenderPaths?.add(builtPage)
     }
   })
+
+  return [...prerenderPaths]
+}
+
+export async function isPageStatic(
+  page: string,
+  serverBundle: string,
+  runtimeEnvConfig: any
+): Promise<{
+  isStatic?: boolean
+  isHybridAmp?: boolean
+  hasServerProps?: boolean
+  hasStaticProps?: boolean
+  prerenderRoutes?: string[] | undefined
+}> {
+  try {
+    require('../next-server/lib/runtime-config').setConfig(runtimeEnvConfig)
+    const mod = require(serverBundle)
+    const Comp = mod.default || mod
+
+    if (!Comp || !isValidElementType(Comp) || typeof Comp === 'string') {
+      throw new Error('INVALID_DEFAULT_EXPORT')
+    }
+
+    const hasGetInitialProps = !!(Comp as any).getInitialProps
+    const hasStaticProps = !!mod.unstable_getStaticProps
+    const hasStaticPaths = !!mod.unstable_getStaticPaths
+    const hasServerProps = !!mod.unstable_getServerProps
+    const hasLegacyStaticParams = !!mod.unstable_getStaticParams
+
+    if (hasLegacyStaticParams) {
+      throw new Error(
+        `unstable_getStaticParams was replaced with unstable_getStaticPaths. Please update your code.`
+      )
+    }
+
+    // A page cannot be prerendered _and_ define a data requirement. That's
+    // contradictory!
+    if (hasGetInitialProps && hasStaticProps) {
+      throw new Error(SSG_GET_INITIAL_PROPS_CONFLICT)
+    }
+
+    if (hasGetInitialProps && hasServerProps) {
+      throw new Error(SERVER_PROPS_GET_INIT_PROPS_CONFLICT)
+    }
+
+    if (hasStaticProps && hasServerProps) {
+      throw new Error(SERVER_PROPS_SSG_CONFLICT)
+    }
+
+    const pageIsDynamic = isDynamicRoute(page)
+    // A page cannot have static parameters if it is not a dynamic page.
+    if (hasStaticProps && hasStaticPaths && !pageIsDynamic) {
+      throw new Error(
+        `unstable_getStaticPaths can only be used with dynamic pages, not '${page}'.` +
+          `\nLearn more: https://nextjs.org/docs#dynamic-routing`
+      )
+    }
+
+    if (hasStaticProps && pageIsDynamic && !hasStaticPaths) {
+      throw new Error(
+        `unstable_getStaticPaths is required for dynamic SSG pages and is missing for '${page}'.` +
+          `\nRead more: https://err.sh/next.js/invalid-getstaticpaths-value`
+      )
+    }
+
+    let prerenderRoutes: Array<string> | undefined
+    if (hasStaticProps && hasStaticPaths) {
+      prerenderRoutes = await buildStaticPaths(
+        page,
+        mod.unstable_getStaticPaths
+      )
     }
 
     const config = mod.config || {}
     return {
       isStatic: !hasStaticProps && !hasGetInitialProps && !hasServerProps,
       isHybridAmp: config.amp === 'hybrid',
-      prerenderRoutes: prerenderPaths && [...prerenderPaths],
+      prerenderRoutes,
       hasStaticProps,
       hasServerProps,
     }

packages/next/next-server/server/next-server.ts

@@ -124,6 +124,9 @@ export default class Server {
     redirects: Redirect[]
     headers: Header[]
   }
+  protected staticPathsWorker?: import('jest-worker').default & {
+    loadStaticPaths: typeof import('../../server/static-paths-worker').loadStaticPaths
+  }
 
   public constructor({
     dir = '.',
@@ -884,6 +887,7 @@ export default class Server {
       typeof (components.Component as any).renderReqToHTML === 'function'
     const isSSG = !!components.unstable_getStaticProps
     const isServerProps = !!components.unstable_getServerProps
+    const hasStaticPaths = !!components.unstable_getStaticPaths
 
     // Toggle whether or not this is a Data request
     const isDataReq = query._nextDataReq
@@ -950,9 +954,10 @@ export default class Server {
     const isPreviewMode = previewData !== false
 
     // Compute the SPR cache key
+    const urlPathname = parseUrl(req.url || '').pathname!
     const ssgCacheKey = isPreviewMode
       ? `__` + nanoid() // Preview mode uses a throw away key to not coalesce preview invokes
-      : parseUrl(req.url || '').pathname!
+      : urlPathname
 
     // Complete the response with cached data if its present
     const cachedData = isPreviewMode
@@ -1020,6 +1025,30 @@ export default class Server {
     const isProduction = !this.renderOpts.dev
     const isDynamicPathname = isDynamicRoute(pathname)
     const didRespond = isResSent(res)
+
+    // we lazy load the staticPaths to prevent the user
+    // from waiting on them for the page to load in dev mode
+    let staticPaths: string[] | undefined
+
+    if (!isProduction && hasStaticPaths) {
+      const __getStaticPaths = async () => {
+        const paths = await this.staticPathsWorker!.loadStaticPaths(
+          this.distDir,
+          this.buildId,
+          pathname,
+          !this.renderOpts.dev && this._isLikeServerless
+        )
+        return paths
+      }
+
+      staticPaths = (
+        await withCoalescedInvoke(__getStaticPaths)(
+          `staticPaths-${pathname}`,
+          []
+        )
+      ).value
+    }
+
     // const isForcedBlocking =
     //   req.headers['X-Prerender-Bypass-Mode'] !== 'Blocking'
 
@@ -1030,20 +1059,20 @@ export default class Server {
     //
     // * Preview mode toggles all pages to be resolved in a blocking manner.
     //
-    // * Non-dynamic pages should block (though this is an be an impossible
+    // * Non-dynamic pages should block (though this is an impossible
     //   case in production).
     //
-    // * Dynamic pages should return their skeleton, then finish the data
-    //   request on the client-side.
+    // * Dynamic pages should return their skeleton if not defined in
+    //   getStaticPaths, then finish the data request on the client-side.
     //
     if (
       !didRespond &&
       !isDataReq &&
       !isPreviewMode &&
       isDynamicPathname &&
-      // TODO: development should trigger fallback when the path is not in
-      // `getStaticPaths`, for now, let's assume it is.
-      isProduction
+      // Development should trigger fallback when the path is not in
+      // `getStaticPaths`
+      (isProduction || !staticPaths || !staticPaths.includes(urlPathname))
     ) {
       let html: string
 

packages/next/server/next-dev-server.ts

@@ -30,6 +30,7 @@ import { Telemetry } from '../telemetry/storage'
 import ErrorDebug from './error-debug'
 import HotReloader from './hot-reloader'
 import { findPageFile } from './lib/find-page-file'
+import Worker from 'jest-worker'
 
 if (typeof React.Suspense === 'undefined') {
   throw new Error(
@@ -79,6 +80,18 @@ export default class DevServer extends Server {
     }
     this.isCustomServer = !options.isNextDevCommand
     this.pagesDir = findPagesDir(this.dir)
+    this.staticPathsWorker = new Worker(
+      require.resolve('./static-paths-worker'),
+      {
+        maxRetries: 0,
+        numWorkers: this.nextConfig.experimental.cpus,
+      }
+    ) as Worker & {
+      loadStaticPaths: typeof import('./static-paths-worker').loadStaticPaths
+    }
+
+    this.staticPathsWorker.getStdout().pipe(process.stdout)
+    this.staticPathsWorker.getStderr().pipe(process.stderr)
   }
 
   protected currentPhase() {

packages/next/server/static-paths-worker.ts

+import { join } from 'path'
+import { buildStaticPaths } from '../build/utils'
+import { getPagePath } from '../next-server/server/require'
+import { loadComponents } from '../next-server/server/load-components'
+import { PAGES_MANIFEST, SERVER_DIRECTORY } from '../next-server/lib/constants'
+
+// we call getStaticPaths in a separate process to ensure
+// side-effects aren't relied on in dev that will break
+// during a production build
+export async function loadStaticPaths(
+  distDir: string,
+  buildId: string,
+  pathname: string,
+  serverless: boolean
+) {
+  // we need to clear any modules manually here since the
+  // require-cache-hot-loader doesn't affect require cache here
+  // since we're in a separate process
+  delete require.cache[join(distDir, SERVER_DIRECTORY, PAGES_MANIFEST)]
+
+  const pagePath = await getPagePath(pathname, distDir, serverless, true)
+  delete require.cache[pagePath]
+
+  const components = await loadComponents(
+    distDir,
+    buildId,
+    pathname,
+    serverless
+  )
+
+  if (!components.unstable_getStaticPaths) {
+    // we shouldn't get to this point since the worker should
+    // only be called for SSG pages with getStaticPaths
+    throw new Error(
+      `Invariant: failed to load page with unstable_getStaticPaths for ${pathname}`
+    )
+  }
+
+  return buildStaticPaths(pathname, components.unstable_getStaticPaths)
+}

test/integration/prerender/test/index.test.js

@@ -381,14 +381,6 @@ const runTests = (dev = false, looseMode = false) => {
   })
 
   it('should support lazy catchall route', async () => {
-    // Dev doesn't support fallback yet
-    if (dev) {
-      const html = await renderViaHTTP(appPort, '/catchall/notreturnedinpaths')
-      const $ = cheerio.load(html)
-      expect($('#catchall').text()).toMatch(/Hi.*?notreturnedinpaths/)
-    }
-    // Production will render fallback for a "lazy" route
-    else {
     const html = await renderViaHTTP(appPort, '/catchall/notreturnedinpaths')
     const $ = cheerio.load(html)
     expect($('#catchall').text()).toBe('fallback')
@@ -403,21 +395,10 @@ const runTests = (dev = false, looseMode = false) => {
       () => browser.elementByCss('#catchall').text(),
       /Hi.*?delayby3s/
     )
-    }
   })
 
   it('should support nested lazy catchall route', async () => {
-    // Dev doesn't support fallback yet
-    if (dev) {
-      const html = await renderViaHTTP(
-        appPort,
-        '/catchall/notreturnedinpaths/nested'
-      )
-      const $ = cheerio.load(html)
-      expect($('#catchall').text()).toMatch(/Hi.*?notreturnedinpaths nested/)
-    }
-    // Production will render fallback for a "lazy" route
-    else {
+    // We will render fallback for a "lazy" route
     const html = await renderViaHTTP(
       appPort,
       '/catchall/notreturnedinpaths/nested'
@@ -435,7 +416,6 @@ const runTests = (dev = false, looseMode = false) => {
       () => browser.elementByCss('#catchall').text(),
       /Hi.*?delayby3s nested/
     )
-    }
   })
 
   if (dev) {
@@ -448,6 +428,28 @@ const runTests = (dev = false, looseMode = false) => {
     //   )
     // })
 
+    it('should always show fallback for page not in getStaticPaths', async () => {
+      const html = await renderViaHTTP(appPort, '/blog/post-321')
+      const $ = cheerio.load(html)
+      expect(JSON.parse($('#__NEXT_DATA__').text()).isFallback).toBe(true)
+
+      // make another request to ensure it still is
+      const html2 = await renderViaHTTP(appPort, '/blog/post-321')
+      const $2 = cheerio.load(html2)
+      expect(JSON.parse($2('#__NEXT_DATA__').text()).isFallback).toBe(true)
+    })
+
+    it('should not show fallback for page in getStaticPaths', async () => {
+      const html = await renderViaHTTP(appPort, '/blog/post-1')
+      const $ = cheerio.load(html)
+      expect(JSON.parse($('#__NEXT_DATA__').text()).isFallback).toBe(false)
+
+      // make another request to ensure it's still not
+      const html2 = await renderViaHTTP(appPort, '/blog/post-1')
+      const $2 = cheerio.load(html2)
+      expect(JSON.parse($2('#__NEXT_DATA__').text()).isFallback).toBe(false)
+    })
+
     it('should log error in console and browser in dev mode', async () => {
       const origContent = await fs.readFile(indexPage, 'utf8')