Make Client request BUILD_ID from the Server (#6891)

* Generate two versions of pages

* Add code VSCode deleted

* Add dynamicBuildId option to __NEXT_DATA__

* Reduce amount of diff

* Make getPageFile code easier to read

* Minimize diff

* minimize diff

* Fix default value for dynamicBuildId

* Fix weird bug

* Fetch the head build id on client

* Move __selectivePageBuilding

* Add tests

* Remove _this

* Add console warning

errors/head-build-id.md

+# Failed to load `BUILD_ID` from Server
+
+#### Why This Error Occurred
+
+The deployment was generated incorrectly or the server was overloaded at the time of the request.
+
+#### Possible Ways to Fix It
+
+Please make sure you are using the latest version of the `@now/next` builder in your `now.json`.
+If this error persists, please file a bug report.

packages/next-server/lib/constants.js

@@ -26,3 +26,4 @@ export const CLIENT_STATIC_FILES_RUNTIME_WEBPACK = `${CLIENT_STATIC_FILES_RUNTIM
 export const IS_BUNDLED_PAGE_REGEX = /^static[/\\][^/\\]+[/\\]pages.*\.js$/
 // matches static/<buildid>/pages/:page*.js
 export const ROUTE_NAME_REGEX = /^static[/\\][^/\\]+[/\\]pages[/\\](.*)\.js$/
+export const HEAD_BUILD_ID_FILE = `${CLIENT_STATIC_FILES_PATH}/HEAD_BUILD_ID`

packages/next-server/server/render.tsx

@@ -116,6 +116,7 @@ type RenderOpts = {
   ampBindInitData: boolean
   staticMarkup: boolean
   buildId: string
+  dynamicBuildId?: boolean
   runtimeConfig?: { [key: string]: any }
   assetPrefix?: string
   err?: Error | null
@@ -143,6 +144,7 @@ function renderDocument(
     pathname,
     query,
     buildId,
+    dynamicBuildId = false,
     assetPrefix,
     runtimeConfig,
     nextExport,
@@ -182,6 +184,7 @@ function renderDocument(
             page: pathname, // The rendered page
             query, // querystring parsed / passed by the user
             buildId, // buildId is used to facilitate caching of page bundles, we send it to the client so that pageloader knows where to load bundles
+            dynamicBuildId, // Specifies if the buildId should by dynamically fetched
             assetPrefix: assetPrefix === '' ? undefined : assetPrefix, // send assetPrefix to the client side when configured, otherwise don't sent in the resulting HTML
             runtimeConfig, // runtimeConfig if provided, otherwise don't sent in the resulting HTML
             nextExport, // If this is a page exported by `next export`

packages/next/build/entries.ts

@@ -32,7 +32,7 @@ type Entrypoints = {
   server: WebpackEntrypoints
 }
 
-export function createEntrypoints(pages: PagesMapping, target: 'server'|'serverless', buildId: string, config: any): Entrypoints {
+export function createEntrypoints(pages: PagesMapping, target: 'server'|'serverless', buildId: string, dynamicBuildId: boolean, config: any): Entrypoints {
   const client: WebpackEntrypoints = {}
   const server: WebpackEntrypoints = {}
 
@@ -44,7 +44,8 @@ export function createEntrypoints(pages: PagesMapping, target: 'server'|'serverl
     assetPrefix: config.assetPrefix,
     generateEtags: config.generateEtags,
     ampEnabled: config.experimental.amp,
-    ampBindInitData: config.experimental.ampBindInitData
+    ampBindInitData: config.experimental.ampBindInitData,
+    dynamicBuildId
   }
 
   Object.keys(pages).forEach((page) => {

packages/next/build/index.ts

@@ -84,8 +84,10 @@ export default async function build(
       ? [process.env.__NEXT_BUILDER_EXPERIMENTAL_PAGE]
       : []
 
+  const __selectivePageBuilding = pages ? Boolean(pages.length) : false
+
   let pagePaths
-  if (pages && pages.length) {
+  if (__selectivePageBuilding) {
     if (config.target !== 'serverless') {
       throw new Error(
         'Cannot use selective page building without the serverless target.'
@@ -125,6 +127,7 @@ export default async function build(
     mappedPages,
     config.target,
     buildId,
+    __selectivePageBuilding,
     config
   )
   const configs = await Promise.all([
@@ -135,7 +138,7 @@ export default async function build(
       config,
       target: config.target,
       entrypoints: entrypoints.client,
-      __selectivePageBuilding: pages && Boolean(pages.length),
+      __selectivePageBuilding,
     }),
     getBaseWebpackConfig(dir, {
       debug,
@@ -144,7 +147,7 @@ export default async function build(
       config,
       target: config.target,
       entrypoints: entrypoints.server,
-      __selectivePageBuilding: pages && Boolean(pages.length),
+      __selectivePageBuilding,
     }),
   ])
 
@@ -203,5 +206,5 @@ export default async function build(
 
   printTreeView(Object.keys(mappedPages))
 
-  await writeBuildId(distDir, buildId)
+  await writeBuildId(distDir, buildId, __selectivePageBuilding)
 }

packages/next/build/webpack-config.ts

@@ -326,7 +326,7 @@ export default function getBaseWebpackConfig (dir: string, {dev = false, debug =
           return /next-server[\\/]dist[\\/]/.test(context) || /next[\\/]dist[\\/]/.test(context)
         }
       }),
-      target === 'serverless' && isServer && new ServerlessPlugin(buildId),
+      target === 'serverless' && (isServer || __selectivePageBuilding) && new ServerlessPlugin(buildId, { isServer }),
       target !== 'serverless' && isServer && new PagesManifestPlugin(),
       target !== 'serverless' && isServer && new NextJsSSRModuleCachePlugin({ outputPath }),
       isServer && new NextJsSsrImportPlugin(),

packages/next/build/webpack/loaders/next-serverless-loader.ts

@@ -14,6 +14,7 @@ export type ServerlessLoaderQuery = {
   ampEnabled: boolean | string,
   ampBindInitData: boolean | string,
   generateEtags: string
+  dynamicBuildId?: string | boolean
 }
 
 const nextServerlessLoader: loader.Loader = function () {
@@ -27,7 +28,8 @@ const nextServerlessLoader: loader.Loader = function () {
     absoluteAppPath,
     absoluteDocumentPath,
     absoluteErrorPath,
-    generateEtags
+    generateEtags,
+    dynamicBuildId
   }: ServerlessLoaderQuery = typeof this.query === 'string' ? parse(this.query.substr(1)) : this.query
   const buildManifest = join(distDir, BUILD_MANIFEST).replace(/\\/g, '/')
   const reactLoadableManifest = join(distDir, REACT_LOADABLE_MANIFEST).replace(/\\/g, '/')
@@ -48,6 +50,7 @@ const nextServerlessLoader: loader.Loader = function () {
         buildManifest,
         reactLoadableManifest,
         buildId: "__NEXT_REPLACE__BUILD_ID__",
+        dynamicBuildId: ${dynamicBuildId === true || dynamicBuildId === 'true'},
         assetPrefix: "${assetPrefix}",
         ampEnabled: ${ampEnabled === true || ampEnabled === 'true'},
         ampBindInitData: ${ampBindInitData === true || ampBindInitData === 'true'}

packages/next/build/webpack/plugins/serverless-plugin.ts

@@ -53,32 +53,52 @@ function interceptFileWrites(
 
 export class ServerlessPlugin {
   private buildId: string
+  private isServer: boolean
 
-  constructor(buildId: string) {
+  constructor(buildId: string, { isServer = false } = {}) {
     this.buildId = buildId
+    this.isServer = isServer
   }
 
   apply(compiler: Compiler) {
-    interceptFileWrites(compiler, content =>
-      replaceInBuffer(content, NEXT_REPLACE_BUILD_ID, this.buildId)
-    )
+    if (this.isServer) {
+      interceptFileWrites(compiler, content =>
+        replaceInBuffer(content, NEXT_REPLACE_BUILD_ID, this.buildId)
+      )
 
-    compiler.hooks.compilation.tap('ServerlessPlugin', compilation => {
-      compilation.hooks.optimizeChunksBasic.tap('ServerlessPlugin', chunks => {
-        chunks.forEach(chunk => {
-          // If chunk is not an entry point skip them
-          if (chunk.hasEntryModule()) {
-            const dynamicChunks = chunk.getAllAsyncChunks()
-            if (dynamicChunks.size !== 0) {
-              for (const dynamicChunk of dynamicChunks) {
-                for (const module of dynamicChunk.modulesIterable) {
-                  GraphHelpers.connectChunkAndModule(chunk, module)
+      compiler.hooks.compilation.tap('ServerlessPlugin', compilation => {
+        compilation.hooks.optimizeChunksBasic.tap(
+          'ServerlessPlugin',
+          chunks => {
+            chunks.forEach(chunk => {
+              // If chunk is not an entry point skip them
+              if (chunk.hasEntryModule()) {
+                const dynamicChunks = chunk.getAllAsyncChunks()
+                if (dynamicChunks.size !== 0) {
+                  for (const dynamicChunk of dynamicChunks) {
+                    for (const module of dynamicChunk.modulesIterable) {
+                      GraphHelpers.connectChunkAndModule(chunk, module)
+                    }
+                  }
                 }
               }
-            }
+            })
           }
-        })
+        )
+      })
+    } else {
+      compiler.hooks.emit.tap('ServerlessPlugin', compilation => {
+        const assetNames = Object.keys(compilation.assets).filter(f =>
+          f.includes(this.buildId)
+        )
+        for (const name of assetNames) {
+          compilation.assets[
+            name
+              .replace(new RegExp(`${this.buildId}[\\/\\\\]`), '')
+              .replace(/[.]js$/, `.${this.buildId}.js`)
+          ] = compilation.assets[name]
+        }
       })
-    })
+    }
   }
 }

packages/next/build/write-build-id.ts

 import fs from 'fs'
 import {promisify} from 'util'
 import {join} from 'path'
-import {BUILD_ID_FILE} from 'next-server/constants'
+import {BUILD_ID_FILE, HEAD_BUILD_ID_FILE} from 'next-server/constants'
 
 const writeFile = promisify(fs.writeFile)
 
-export async function writeBuildId (distDir: string, buildId: string): Promise<void> {
+export async function writeBuildId (distDir: string, buildId: string, headBuildId: boolean): Promise<void> {
   const buildIdPath = join(distDir, BUILD_ID_FILE)
   await writeFile(buildIdPath, buildId, 'utf8')
+
+  if (headBuildId) {
+    const headBuildIdPath = join(distDir, HEAD_BUILD_ID_FILE)
+    await writeFile(headBuildIdPath, buildId, 'utf8')
+  }
 }

packages/next/client/index.js

@@ -31,6 +31,7 @@ const {
   page,
   query,
   buildId,
+  dynamicBuildId,
   assetPrefix,
   runtimeConfig,
   dynamicIds
@@ -99,6 +100,10 @@ export default async ({
 
   await Loadable.preloadReady(dynamicIds || [])
 
+  if (dynamicBuildId === true) {
+    pageLoader.onDynamicBuildId()
+  }
+
   router = createRouter(page, query, asPath, {
     initialProps: props,
     pageLoader,

packages/next/client/page-loader.js

 /* global document */
 import mitt from 'next-server/dist/lib/mitt'
+import unfetch from 'unfetch'
 
 // smaller version of https://gist.github.com/igrigorik/a02f2359f3bc50ca7a9c
 function supportsPreload (list) {
@@ -24,6 +25,7 @@ export default class PageLoader {
     this.prefetchCache = new Set()
     this.pageRegisterEvents = mitt()
     this.loadingRoutes = {}
+    this.promisedBuildId = Promise.resolve()
   }
 
   normalizeRoute (route) {
@@ -76,7 +78,38 @@ export default class PageLoader {
     })
   }
 
-  loadScript (route) {
+  onDynamicBuildId () {
+    this.promisedBuildId = new Promise(resolve => {
+      unfetch(`${this.assetPrefix}/_next/static/HEAD_BUILD_ID`)
+        .then(res => {
+          if (res.ok) {
+            return res
+          }
+
+          const err = new Error('Failed to fetch HEAD buildId')
+          err.res = res
+          throw err
+        })
+        .then(res => res.text())
+        .then(buildId => {
+          this.buildId = buildId.trim()
+        })
+        .catch(() => {
+          // When this fails it's not a _huge_ deal, preload wont work and page
+          // navigation will 404, triggering a SSR refresh
+          console.warn(
+            'Failed to load BUILD_ID from server. ' +
+              'The following client-side page transition will likely 404 and cause a SSR.\n' +
+              'http://err.sh/zeit/next.js/head-build-id'
+          )
+        })
+        .then(resolve, resolve)
+    })
+  }
+
+  async loadScript (route) {
+    await this.promisedBuildId
+
     route = this.normalizeRoute(route)
     const scriptRoute = route === '/' ? '/index.js' : `${route}.js`
 
@@ -146,6 +179,8 @@ export default class PageLoader {
     // If not fall back to loading script tags before the page is loaded
     // https://caniuse.com/#feat=link-rel-preload
     if (hasPreload) {
+      await this.promisedBuildId
+
       const link = document.createElement('link')
       link.rel = 'preload'
       link.crossOrigin = process.crossOrigin

packages/next/package.json

@@ -116,7 +116,7 @@
     "@types/webpack-sources": "0.1.5",
     "@zeit/ncc": "0.15.2",
     "arg": "4.1.0",
-    "nanoid": "1.2.1",
+    "nanoid": "2.0.1",
     "resolve": "1.5.0",
     "taskr": "1.1.0",
     "text-table": "0.2.0",

packages/next/pages/_document.js

@@ -157,7 +157,7 @@ export class Head extends Component {
       __NEXT_DATA__,
     } = this.context._documentProps
     const { _devOnlyInvalidateCacheQueryString } = this.context
-    const { page, buildId } = __NEXT_DATA__
+    const { page, buildId, dynamicBuildId } = __NEXT_DATA__
     const isDirtyAmp = amphtml && !__NEXT_DATA__.query.amp
 
     let { head } = this.context._documentProps
@@ -255,9 +255,13 @@ export class Head extends Component {
             {page !== '/_error' && (
               <link
                 rel="preload"
-                href={`${assetPrefix}/_next/static/${buildId}/pages${getPagePathname(
-                  page
-                )}${_devOnlyInvalidateCacheQueryString}`}
+                href={
+                  assetPrefix +
+                  (dynamicBuildId
+                    ? `/_next/static/pages${getPageFile(page, buildId)}`
+                    : `/_next/static/${buildId}/pages${getPageFile(page)}`) +
+                  _devOnlyInvalidateCacheQueryString
+                }
                 as="script"
                 nonce={this.props.nonce}
                 crossOrigin={this.props.crossOrigin || process.crossOrigin}
@@ -265,7 +269,13 @@ export class Head extends Component {
             )}
             <link
               rel="preload"
-              href={`${assetPrefix}/_next/static/${buildId}/pages/_app.js${_devOnlyInvalidateCacheQueryString}`}
+              href={
+                assetPrefix +
+                (dynamicBuildId
+                  ? `/_next/static/pages/_app.${buildId}.js`
+                  : `/_next/static/${buildId}/pages/_app.js`) +
+                _devOnlyInvalidateCacheQueryString
+              }
               as="script"
               nonce={this.props.nonce}
               crossOrigin={this.props.crossOrigin || process.crossOrigin}
@@ -416,7 +426,7 @@ export class NextScript extends Component {
       )
     }
 
-    const { page, buildId } = __NEXT_DATA__
+    const { page, buildId, dynamicBuildId } = __NEXT_DATA__
 
     if (process.env.NODE_ENV !== 'production') {
       if (this.props.crossOrigin)
@@ -454,9 +464,13 @@ export class NextScript extends Component {
           <script
             async
             id={`__NEXT_PAGE__${page}`}
-            src={`${assetPrefix}/_next/static/${buildId}/pages${getPagePathname(
-              page
-            )}${_devOnlyInvalidateCacheQueryString}`}
+            src={
+              assetPrefix +
+              (dynamicBuildId
+                ? `/_next/static/pages${getPageFile(page, buildId)}`
+                : `/_next/static/${buildId}/pages${getPageFile(page)}`) +
+              _devOnlyInvalidateCacheQueryString
+            }
             nonce={this.props.nonce}
             crossOrigin={this.props.crossOrigin || process.crossOrigin}
           />
@@ -464,7 +478,13 @@ export class NextScript extends Component {
         <script
           async
           id={`__NEXT_PAGE__/_app`}
-          src={`${assetPrefix}/_next/static/${buildId}/pages/_app.js${_devOnlyInvalidateCacheQueryString}`}
+          src={
+            assetPrefix +
+            (dynamicBuildId
+              ? `/_next/static/pages/_app.${buildId}.js`
+              : `/_next/static/${buildId}/pages/_app.js`) +
+            _devOnlyInvalidateCacheQueryString
+          }
           nonce={this.props.nonce}
           crossOrigin={this.props.crossOrigin || process.crossOrigin}
         />
@@ -475,10 +495,10 @@ export class NextScript extends Component {
   }
 }
 
-function getPagePathname(page) {
+function getPageFile(page, buildId) {
   if (page === '/') {
-    return '/index.js'
+    return buildId ? `/index.${buildId}.js` : '/index.js'
   }
 
-  return `${page}.js`
+  return buildId ? `${page}.${buildId}.js` : `${page}.js`
 }

packages/next/server/hot-reloader.js

@@ -170,7 +170,7 @@ export default class HotReloader {
     ])
 
     const pages = createPagesMapping(pagePaths.filter(i => i !== null), this.config.pageExtensions)
-    const entrypoints = createEntrypoints(pages, 'server', this.buildId, this.config)
+    const entrypoints = createEntrypoints(pages, 'server', this.buildId, false, this.config)
 
     let additionalClientEntrypoints = {}
     if (this.config.experimental.amp) {

test/integration/serverless-on-demand-pages/next.config.js

+module.exports = {
+  target: 'serverless',
+  onDemandEntries: {
+    // Make sure entries are not getting disposed.
+    maxInactiveAge: 1000 * 60 * 60
+  },
+  lambdas: true
+}

test/integration/serverless-on-demand-pages/pages/abc.js

+export default () => <div>test</div>

test/integration/serverless-on-demand-pages/pages/fetch.js

+import fetch from 'isomorphic-unfetch'
+import React from 'react'
+
+export default class extends React.Component {
+  static async getInitialProps () {
+    try {
+      const res = await fetch('')
+      const text = await res.text()
+      console.log(text)
+      return { text }
+    } catch (err) {
+      if (err.message.includes('is not a function')) {
+        return { failed: true, error: err.toString() }
+      }
+
+      return { error: err.toString() }
+    }
+  }
+  render () {
+    const { failed, error, text } = this.props
+    return (
+      <div className='fetch-page'>
+        {failed ? 'failed' : ''}
+        {error}
+        <div id='text'>{text}</div>
+      </div>
+    )
+  }
+}

test/integration/serverless-on-demand-pages/pages/index.js

+import Link from 'next/link'
+export default () => {
+  return (
+    <div>
+      Hello World
+      <Link href='/fetch'>
+        <a id='fetchlink'>fetch page</a>
+      </Link>
+    </div>
+  )
+}

test/integration/serverless-on-demand-pages/run-server.js

+const start = require('./server')
+start(3000).then(() => console.log('http://localhost:3000'))

test/integration/serverless-on-demand-pages/server.js

+const express = require('express')
+const http = require('http')
+const path = require('path')
+
+module.exports = function start (port = 0) {
+  return new Promise((resolve, reject) => {
+    const app = express()
+    const nextStaticDir = path.join(__dirname, '.next', 'static')
+    app.use('/_next/static', express.static(nextStaticDir))
+    app.get('/', (req, res) => {
+      require('./.next/serverless/pages/index.js').render(req, res)
+    })
+    app.get('/fetch', (req, res) => {
+      require('./.next/serverless/pages/fetch.js').render(req, res)
+    })
+    app.get('/404', (req, res) => {
+      require('./.next/serverless/pages/_error.js').render(req, res)
+    })
+    const server = new http.Server(app)
+
+    server.listen(port, err => {
+      if (err) {
+        return reject(err)
+      }
+
+      resolve(server)
+    })
+  })
+}

test/integration/serverless-on-demand-pages/test/index.test.js

+/* eslint-env jest */
+/* global jasmine, test */
+import webdriver from 'next-webdriver'
+import { join } from 'path'
+import { existsSync } from 'fs'
+import { nextBuild, stopApp, renderViaHTTP } from 'next-test-utils'
+import startServer from '../server'
+import fetch from 'node-fetch'
+
+const appDir = join(__dirname, '../')
+let appPort
+let server
+jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000 * 60 * 5
+
+describe('Serverless', () => {
+  beforeAll(async () => {
+    await nextBuild(appDir, [
+      '--experimental-page',
+      '/',
+      '--experimental-page',
+      '/fetch'
+    ])
+    server = await startServer()
+    appPort = server.address().port
+  })
+  afterAll(() => stopApp(server))
+
+  it('should render the page', async () => {
+    const html = await renderViaHTTP(appPort, '/')
+    expect(html).toMatch(/Hello World/)
+  })
+
+  it('should render correctly when importing isomorphic-unfetch', async () => {
+    const url = `http://localhost:${appPort}/fetch`
+    const res = await fetch(url)
+    expect(res.status).toBe(200)
+    const text = await res.text()
+    expect(text.includes('failed')).toBe(false)
+  })
+
+  it('should render correctly when importing isomorphic-unfetch on the client side', async () => {
+    const browser = await webdriver(appPort, '/')
+    try {
+      const text = await browser
+        .elementByCss('a')
+        .click()
+        .waitForElementByCss('.fetch-page')
+        .elementByCss('#text')
+        .text()
+
+      expect(text).toMatch(/fetch page/)
+    } finally {
+      await browser.close()
+    }
+  })
+
+  it('should not output abc.js to serverless build', () => {
+    const serverlessDir = join(appDir, '.next/serverless/pages')
+    expect(existsSync(join(serverlessDir, 'abc.js'))).toBeFalsy()
+  })
+})

yarn.lock

@@ -8626,10 +8626,10 @@ nan@^2.9.2:
   resolved "https://registry.yarnpkg.com/nan/-/nan-2.12.1.tgz#7b1aa193e9aa86057e3c7bbd0ac448e770925552"
   integrity sha512-JY7V6lRkStKcKTvHO5NVSQRv+RV+FIL5pvDoLiAtSL9pKlC5x9PKQcZDsq7m4FO4d57mkhC6Z+QhAh3Jdk5JFw==
 
-nanoid@1.2.1:
-  version "1.2.1"
-  resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-1.2.1.tgz#922bf6c10e35f7b208993768dad643577c907adf"
-  integrity sha512-S1QSG+TQtsqr2/ujHZcNT0OxygffUaUT755qTc/SPKfQ0VJBlOO6qb1425UYoHXPvCZ3pWgMVCuy1t7+AoCxnQ==
+nanoid@2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-2.0.1.tgz#deb55cac196e3f138071911dabbc3726eb048864"
+  integrity sha512-k1u2uemjIGsn25zmujKnotgniC/gxQ9sdegdezeDiKdkDW56THUMqlz3urndKCXJxA6yPzSZbXx/QCMe/pxqsA==
 
 nanomatch@^1.2.9:
   version "1.2.13"