Add buildId to SPR data routes (#8929)

* Add buildId to SPR data routes

* Update buildId replace in serverless loader

* Use new RegExp and add comment

* Apply suggestions from code review
Co-Authored-By: NJoe Haddad <joe.haddad@zeit.co>

* Test to ensure client transition and handle / data route

packages/next/build/index.ts

@@ -535,6 +535,7 @@ export default async function build(dir: string, conf = null): Promise<void> {
             srcRoute: null,
             dataRoute: path.posix.join(
               '/_next/data',
+              buildId,
               `${page === '/' ? '/index' : page}.json`
             ),
           }
@@ -552,6 +553,7 @@ export default async function build(dir: string, conf = null): Promise<void> {
               srcRoute: page,
               dataRoute: path.posix.join(
                 '/_next/data',
+                buildId,
                 `${route === '/' ? '/index' : route}.json`
               ),
             }
@@ -583,6 +585,7 @@ export default async function build(dir: string, conf = null): Promise<void> {
     tbdPrerenderRoutes.forEach(tbdRoute => {
       const dataRoute = path.posix.join(
         '/_next/data',
+        buildId,
         `${tbdRoute === '/' ? '/index' : tbdRoute}.json`
       )
 

packages/next/build/webpack/loaders/next-serverless-loader.ts

@@ -42,6 +42,7 @@ const nextServerlessLoader: loader.Loader = function() {
     /\\/g,
     '/'
   )
+  const escapedBuildId = buildId.replace(/[|\\{}()[\]^$+*?.-]/g, '\\$&')
 
   if (page.match(API_ROUTE)) {
     return `
@@ -110,7 +111,7 @@ const nextServerlessLoader: loader.Loader = function() {
       if (req.url.match(/_next\\/data/)) {
         sprData = true
         req.url = req.url
-          .replace(/\\/_next\\/data\\//, '/')
+          .replace(new RegExp('/_next/data/${escapedBuildId}/'), '/')
           .replace(/\\.json$/, '')
       }
       const parsedUrl = parse(req.url, true)

packages/next/export/index.ts

@@ -243,7 +243,9 @@ export default async function(
   }
 
   const progress = !options.silent && createProgress(filteredPaths.length)
-  const sprDataDir = options.buildExport ? outDir : join(outDir, '_next/data')
+  const sprDataDir = options.buildExport
+    ? outDir
+    : join(outDir, '_next/data', buildId)
 
   const ampValidations: AmpPageStatus = {}
   let hadValidationError = false

packages/next/next-server/lib/router/router.ts

@@ -623,9 +623,14 @@ export default class Router implements BaseRouter {
       (Component as any).__NEXT_SPR
     ) {
       let status: any
-      const { pathname } = parse(ctx.asPath || ctx.pathname)
+      // pathname should have leading slash
+      let { pathname } = parse(ctx.asPath || ctx.pathname)
+      pathname = !pathname || pathname === '/' ? '/index' : pathname
 
-      props = await fetch(`/_next/data${pathname}.json`)
+      props = await fetch(
+        // @ts-ignore __NEXT_DATA__
+        `/_next/data/${__NEXT_DATA__.buildId}${pathname}.json`
+      )
         .then(res => {
           if (!res.ok) {
             status = res.status

packages/next/next-server/server/next-server.ts

@@ -252,10 +252,24 @@ export default class Server {
       {
         match: route('/_next/data/:path*'),
         fn: async (req, res, params, _parsedUrl) => {
-          // Make sure to 404 for /_next/data/ itself
-          if (!params.path) return this.render404(req, res, _parsedUrl)
-          // TODO: force `.json` to be present
-          const pathname = `/${params.path.join('/')}`.replace(/\.json$/, '')
+          // Make sure to 404 for /_next/data/ itself and
+          // we also want to 404 if the buildId isn't correct
+          if (!params.path || params.path[0] !== this.buildId) {
+            return this.render404(req, res, _parsedUrl)
+          }
+          // remove buildId from URL
+          params.path.shift()
+
+          // show 404 if it doesn't end with .json
+          if (!params.path[params.path.length - 1].endsWith('.json')) {
+            return this.render404(req, res, _parsedUrl)
+          }
+
+          // re-create page's pathname
+          const pathname = `/${params.path.join('/')}`
+            .replace(/\.json$/, '')
+            .replace(/\/index$/, '/')
+
           req.url = pathname
           const parsedUrl = parseUrl(pathname, true)
           await this.render(
@@ -601,8 +615,9 @@ export default class Server {
     // Serverless requests need its URL transformed back into the original
     // request path (to emulate lambda behavior in production)
     if (isLikeServerless && isSprData) {
-      const curUrl = parseUrl(req.url || '', true)
-      req.url = `/_next/data${curUrl.pathname}.json`
+      let { pathname } = parseUrl(req.url || '', true)
+      pathname = !pathname || pathname === '/' ? '/index' : pathname
+      req.url = `/_next/data/${this.buildId}${pathname}.json`
     }
 
     const doRender = withCoalescedInvoke(async function(): Promise<{

test/integration/prerender/test/index.test.js

@@ -20,9 +20,58 @@ const appDir = join(__dirname, '..')
 const nextConfig = join(appDir, 'next.config.js')
 let app
 let appPort
+let buildId
 let distPagesDir
 let exportDir
 
+const expectedManifestRoutes = () => ({
+  '/': {
+    dataRoute: `/_next/data/${buildId}/index.json`,
+    initialRevalidateSeconds: 1,
+    srcRoute: null
+  },
+  '/blog/[post3]': {
+    dataRoute: `/_next/data/${buildId}/blog/[post3].json`,
+    initialRevalidateSeconds: 10,
+    srcRoute: '/blog/[post]'
+  },
+  '/blog/post-1': {
+    dataRoute: `/_next/data/${buildId}/blog/post-1.json`,
+    initialRevalidateSeconds: 10,
+    srcRoute: '/blog/[post]'
+  },
+  '/blog/post-2': {
+    dataRoute: `/_next/data/${buildId}/blog/post-2.json`,
+    initialRevalidateSeconds: 10,
+    srcRoute: '/blog/[post]'
+  },
+  '/blog/post-1/comment-1': {
+    dataRoute: `/_next/data/${buildId}/blog/post-1/comment-1.json`,
+    initialRevalidateSeconds: 2,
+    srcRoute: '/blog/[post]/[comment]'
+  },
+  '/blog/post-2/comment-2': {
+    dataRoute: `/_next/data/${buildId}/blog/post-2/comment-2.json`,
+    initialRevalidateSeconds: 2,
+    srcRoute: '/blog/[post]/[comment]'
+  },
+  '/another': {
+    dataRoute: `/_next/data/${buildId}/another.json`,
+    initialRevalidateSeconds: 0,
+    srcRoute: null
+  },
+  '/default-revalidate': {
+    dataRoute: `/_next/data/${buildId}/default-revalidate.json`,
+    initialRevalidateSeconds: 1,
+    srcRoute: null
+  },
+  '/something': {
+    dataRoute: `/_next/data/${buildId}/something.json`,
+    initialRevalidateSeconds: false,
+    srcRoute: null
+  }
+})
+
 const navigateTest = () => {
   it('should navigate between pages successfully', async () => {
     const browser = await webdriver(appPort, '/')
@@ -36,16 +85,19 @@ const navigateTest = () => {
     expect(text).toMatch(/hello.*?world/)
 
     // go to /
+    await browser.eval('window.didTransition = 1')
     await browser.elementByCss('#home').click()
     await browser.waitForElementByCss('#another')
     text = await browser.elementByCss('p').text()
     expect(text).toMatch(/hello.*?world/)
+    expect(await browser.eval('window.didTransition')).toBe(1)
 
     // go to /something
     await browser.elementByCss('#something').click()
     await browser.waitForElementByCss('#home')
     text = await browser.elementByCss('p').text()
     expect(text).toMatch(/hello.*?world/)
+    expect(await browser.eval('window.didTransition')).toBe(1)
 
     // go to /
     await browser.elementByCss('#home').click()
@@ -56,6 +108,7 @@ const navigateTest = () => {
     await browser.waitForElementByCss('#home')
     text = await browser.elementByCss('p').text()
     expect(text).toMatch(/Post:.*?post-1/)
+    expect(await browser.eval('window.didTransition')).toBe(1)
 
     // go to /
     await browser.elementByCss('#home').click()
@@ -66,6 +119,7 @@ const navigateTest = () => {
     await browser.waitForElementByCss('#home')
     text = await browser.elementByCss('p:nth-child(2)').text()
     expect(text).toMatch(/Comment:.*?comment-1/)
+    expect(await browser.eval('window.didTransition')).toBe(1)
 
     await browser.close()
   })
@@ -86,21 +140,33 @@ const runTests = (dev = false) => {
 
   it('should return data correctly', async () => {
     const data = JSON.parse(
-      await renderViaHTTP(appPort, '/_next/data/something.json')
+      await renderViaHTTP(
+        appPort,
+        expectedManifestRoutes()['/something'].dataRoute
+      )
     )
     expect(data.pageProps.world).toBe('world')
   })
 
   it('should return data correctly for dynamic page', async () => {
     const data = JSON.parse(
-      await renderViaHTTP(appPort, '/_next/data/blog/post-1.json')
+      await renderViaHTTP(
+        appPort,
+        expectedManifestRoutes()['/blog/post-1'].dataRoute
+      )
     )
     expect(data.pageProps.post).toBe('post-1')
   })
 
   it('should return data correctly for dynamic page (non-seeded)', async () => {
     const data = JSON.parse(
-      await renderViaHTTP(appPort, '/_next/data/blog/post-3.json')
+      await renderViaHTTP(
+        appPort,
+        expectedManifestRoutes()['/blog/post-1'].dataRoute.replace(
+          /post-1/,
+          'post-3'
+        )
+      )
     )
     expect(data.pageProps.post).toBe('post-3')
   })
@@ -147,66 +213,22 @@ const runTests = (dev = false) => {
     })
   } else {
     it('outputs a prerender-manifest correctly', async () => {
-      const manifest = require(join(appDir, '.next', 'prerender-manifest.json'))
+      const manifest = JSON.parse(
+        await fs.readFile(join(appDir, '.next/prerender-manifest.json'), 'utf8')
+      )
+      const escapedBuildId = buildId.replace(/[|\\{}()[\]^$+*?.-]/g, '\\$&')
 
       expect(manifest.version).toBe(1)
-      expect(manifest.routes).toEqual({
-        '/': {
-          dataRoute: '/_next/data/index.json',
-          initialRevalidateSeconds: 1,
-          srcRoute: null
-        },
-        '/blog/[post3]': {
-          dataRoute: '/_next/data/blog/[post3].json',
-          initialRevalidateSeconds: 10,
-          srcRoute: '/blog/[post]'
-        },
-        '/blog/post-1': {
-          dataRoute: '/_next/data/blog/post-1.json',
-          initialRevalidateSeconds: 10,
-          srcRoute: '/blog/[post]'
-        },
-        '/blog/post-2': {
-          dataRoute: '/_next/data/blog/post-2.json',
-          initialRevalidateSeconds: 10,
-          srcRoute: '/blog/[post]'
-        },
-        '/blog/post-1/comment-1': {
-          dataRoute: '/_next/data/blog/post-1/comment-1.json',
-          initialRevalidateSeconds: 2,
-          srcRoute: '/blog/[post]/[comment]'
-        },
-        '/blog/post-2/comment-2': {
-          dataRoute: '/_next/data/blog/post-2/comment-2.json',
-          initialRevalidateSeconds: 2,
-          srcRoute: '/blog/[post]/[comment]'
-        },
-        '/another': {
-          dataRoute: '/_next/data/another.json',
-          initialRevalidateSeconds: 0,
-          srcRoute: null
-        },
-        '/default-revalidate': {
-          dataRoute: '/_next/data/default-revalidate.json',
-          initialRevalidateSeconds: 1,
-          srcRoute: null
-        },
-        '/something': {
-          dataRoute: '/_next/data/something.json',
-          initialRevalidateSeconds: false,
-          srcRoute: null
-        }
-      })
+      expect(manifest.routes).toEqual(expectedManifestRoutes())
       expect(manifest.dynamicRoutes).toEqual({
         '/blog/[post]': {
-          dataRoute: '/_next/data/blog/[post].json',
-          dataRouteRegex: '^\\/_next\\/data\\/blog\\/([^\\/]+?)\\.json$',
+          dataRoute: `/_next/data/${buildId}/blog/[post].json`,
+          dataRouteRegex: `^\\/_next\\/data\\/${escapedBuildId}\\/blog\\/([^\\/]+?)\\.json$`,
           routeRegex: '^\\/blog\\/([^\\/]+?)(?:\\/)?$'
         },
         '/blog/[post]/[comment]': {
-          dataRoute: '/_next/data/blog/[post]/[comment].json',
-          dataRouteRegex:
-            '^\\/_next\\/data\\/blog\\/([^\\/]+?)\\/([^\\/]+?)\\.json$',
+          dataRoute: `/_next/data/${buildId}/blog/[post]/[comment].json`,
+          dataRouteRegex: `^\\/_next\\/data\\/${escapedBuildId}\\/blog\\/([^\\/]+?)\\/([^\\/]+?)\\.json$`,
           routeRegex: '^\\/blog\\/([^\\/]+?)\\/([^\\/]+?)(?:\\/)?$'
         }
       })
@@ -254,7 +276,7 @@ const runTests = (dev = false) => {
     })
 
     it('should handle revalidating JSON correctly', async () => {
-      const route = '/_next/data/blog/post-2/comment-3'
+      const route = `/_next/data/${buildId}/blog/post-2/comment-3.json`
       const initialJson = await renderViaHTTP(appPort, route)
       expect(initialJson).toMatch(/post-2/)
       expect(initialJson).toMatch(/comment-3/)
@@ -276,6 +298,7 @@ describe('SPR Prerender', () => {
     beforeAll(async () => {
       appPort = await findPort()
       app = await launchApp(appDir, appPort)
+      buildId = 'development'
     })
     afterAll(() => killApp(app))
 
@@ -293,6 +316,7 @@ describe('SPR Prerender', () => {
       appPort = await findPort()
       app = await nextStart(appDir, appPort)
       distPagesDir = join(appDir, '.next/serverless/pages')
+      buildId = await fs.readFile(join(appDir, '.next/BUILD_ID'), 'utf8')
     })
     afterAll(() => killApp(app))
 
@@ -307,7 +331,7 @@ describe('SPR Prerender', () => {
       await nextBuild(appDir)
       appPort = await findPort()
       app = await nextStart(appDir, appPort)
-      const buildId = await fs.readFile(join(appDir, '.next/BUILD_ID'), 'utf8')
+      buildId = await fs.readFile(join(appDir, '.next/BUILD_ID'), 'utf8')
       distPagesDir = join(appDir, '.next/server/static', buildId, 'pages')
     })
     afterAll(() => killApp(app))
@@ -322,6 +346,7 @@ describe('SPR Prerender', () => {
       await nextExport(appDir, { outdir: exportDir })
       app = await startStaticServer(exportDir)
       appPort = app.address().port
+      buildId = await fs.readFile(join(appDir, '.next/BUILD_ID'), 'utf8')
     })
     afterAll(() => killApp(app))
 
@@ -335,7 +360,7 @@ describe('SPR Prerender', () => {
 
       for (const route of routes) {
         await fs.access(join(exportDir, `${route}.html`))
-        await fs.access(join(exportDir, '_next/data', `${route}.json`))
+        await fs.access(join(exportDir, '_next/data', buildId, `${route}.json`))
       }
     })