Emit manifest of all page files (#6853)

* Get all modules included in build

* Add tests

* Get all modules contained per entry chunk

* Sort files

* Add specialized page entry to manifest

* Split manifest into pages and chunks key

* Update test

* Use relative paths to build directory

* Update test

packages/next-server/lib/constants.js

@@ -5,6 +5,7 @@ export const PHASE_DEVELOPMENT_SERVER = 'phase-development-server'
 export const PAGES_MANIFEST = 'pages-manifest.json'
 export const BUILD_MANIFEST = 'build-manifest.json'
 export const REACT_LOADABLE_MANIFEST = 'react-loadable-manifest.json'
+export const CHUNK_GRAPH_MANIFEST = 'compilation-modules.json'
 export const SERVER_DIRECTORY = 'server'
 export const CONFIG_FILE = 'next.config.js'
 export const BUILD_ID_FILE = 'BUILD_ID'

packages/next/build/webpack-config.ts

@@ -7,13 +7,14 @@ import PagesManifestPlugin from './webpack/plugins/pages-manifest-plugin'
 import BuildManifestPlugin from './webpack/plugins/build-manifest-plugin'
 import ChunkNamesPlugin from './webpack/plugins/chunk-names-plugin'
 import { ReactLoadablePlugin } from './webpack/plugins/react-loadable-plugin'
-import { SERVER_DIRECTORY, REACT_LOADABLE_MANIFEST, CLIENT_STATIC_FILES_RUNTIME_WEBPACK, CLIENT_STATIC_FILES_RUNTIME_MAIN } from 'next-server/constants'
+import { SERVER_DIRECTORY, REACT_LOADABLE_MANIFEST, CHUNK_GRAPH_MANIFEST, CLIENT_STATIC_FILES_RUNTIME_WEBPACK, CLIENT_STATIC_FILES_RUNTIME_MAIN } from 'next-server/constants'
 import { NEXT_PROJECT_ROOT, NEXT_PROJECT_ROOT_NODE_MODULES, NEXT_PROJECT_ROOT_DIST_CLIENT, PAGES_DIR_ALIAS, DOT_NEXT_ALIAS } from '../lib/constants'
 import {TerserPlugin} from './webpack/plugins/terser-webpack-plugin/src/index'
 import { ServerlessPlugin } from './webpack/plugins/serverless-plugin'
 import { AllModulesIdentifiedPlugin } from './webpack/plugins/all-modules-identified-plugin'
 import { SharedRuntimePlugin } from './webpack/plugins/shared-runtime-plugin'
 import { HashedChunkIdsPlugin } from './webpack/plugins/hashed-chunk-ids-plugin'
+import { ChunkGraphPlugin } from './webpack/plugins/chunk-graph-plugin'
 import { WebpackEntrypoints } from './entries'
 type ExcludesFalse = <T>(x: T | false) => x is T
 
@@ -273,6 +274,7 @@ export default function getBaseWebpackConfig (dir: string, {dev = false, isServe
       !isServer && new ReactLoadablePlugin({
         filename: REACT_LOADABLE_MANIFEST
       }),
+      !isServer && new ChunkGraphPlugin(path.resolve(dir), { filename: CHUNK_GRAPH_MANIFEST }),
       ...(dev ? (() => {
         // Even though require.cache is server only we have to clear assets from both compilations
         // This is because the client compilation generates the build manifest that's used on the server side

packages/next/build/webpack/plugins/chunk-graph-plugin.ts

+import { Compiler, Plugin } from 'webpack'
+import path from 'path'
+import { EOL } from 'os'
+import { parse } from 'querystring'
+import { CLIENT_STATIC_FILES_RUNTIME_MAIN } from 'next-server/constants'
+
+function getFiles(dir: string, modules: any[]): string[] {
+  if (!(modules && modules.length)) {
+    return []
+  }
+
+  function getFileByIdentifier(id: string) {
+    if (id.startsWith('external ') || id.startsWith('multi ')) {
+      return null
+    }
+
+    let n
+    if ((n = id.lastIndexOf('!')) !== -1) {
+      id = id.substring(n + 1)
+    }
+
+    if (id && !path.isAbsolute(id)) {
+      id = path.resolve(dir, id)
+    }
+
+    return id
+  }
+
+  return modules
+    .reduce(
+      (acc: any[], val: any) =>
+        val.modules
+          ? acc.concat(getFiles(dir, val.modules))
+          : (acc.push(
+              getFileByIdentifier(
+                typeof val.identifier === 'function'
+                  ? val.identifier()
+                  : val.identifier
+              )
+            ),
+            acc),
+      []
+    )
+    .filter(Boolean)
+}
+
+export class ChunkGraphPlugin implements Plugin {
+  private dir: string
+  private filename: string
+
+  constructor(dir: string, { filename }: { filename?: string } = {}) {
+    this.dir = dir
+    this.filename = filename || 'chunk-graph-manifest.json'
+  }
+
+  apply(compiler: Compiler) {
+    const { dir } = this
+    compiler.hooks.emit.tap('ChunkGraphPlugin', compilation => {
+      type StringDictionary = { [pageName: string]: string[] }
+      const manifest: { pages: StringDictionary; chunks: StringDictionary } = {
+        pages: {},
+        chunks: {},
+      }
+
+      let clientRuntime = [] as string[]
+      const pages: StringDictionary = {}
+
+      compilation.chunks.forEach(chunk => {
+        if (!chunk.hasEntryModule()) {
+          return
+        }
+
+        const chunkModules = new Map<any, any>()
+
+        const queue = new Set<any>(chunk.groupsIterable)
+        const chunksProcessed = new Set<any>()
+
+        for (const chunkGroup of queue) {
+          for (const chunk of chunkGroup.chunks) {
+            if (!chunksProcessed.has(chunk)) {
+              chunksProcessed.add(chunk)
+              for (const m of chunk.modulesIterable) {
+                chunkModules.set(m.id, m)
+              }
+            }
+          }
+          for (const child of chunkGroup.childrenIterable) {
+            queue.add(child)
+          }
+        }
+
+        const modules = [...chunkModules.values()]
+        const files = getFiles(dir, modules)
+          .filter(val => !val.includes('node_modules'))
+          .map(f => path.relative(dir, f))
+          .sort()
+
+        let pageName: string | undefined
+        if (chunk.entryModule && chunk.entryModule.loaders) {
+          const entryLoader = chunk.entryModule.loaders.find(
+            ({
+              loader,
+              options,
+            }: {
+              loader?: string | null
+              options?: string | null
+            }) =>
+              loader && loader.includes('next-client-pages-loader') && options
+          )
+          if (entryLoader) {
+            const { page } = parse(entryLoader.options)
+            if (typeof page === 'string' && page) {
+              pageName = page
+            }
+          }
+        }
+
+        if (pageName) {
+          pages[pageName] = files
+        } else {
+          if (chunk.name === CLIENT_STATIC_FILES_RUNTIME_MAIN) {
+            clientRuntime = files
+          } else {
+            manifest.chunks[chunk.name] = files
+          }
+        }
+
+        for (const page in pages) {
+          manifest.pages[page] = [...pages[page], ...clientRuntime]
+        }
+      })
+
+      const json = JSON.stringify(manifest, null, 2) + EOL
+      compilation.assets[this.filename] = {
+        source() {
+          return json
+        },
+        size() {
+          return json.length
+        },
+      }
+    })
+  }
+}

test/integration/dist-dir/test/index.test.js

 /* eslint-env jest */
 /* global jasmine */
-import { join } from 'path'
-import { existsSync } from 'fs'
-import { BUILD_ID_FILE } from 'next-server/constants'
+import { join, resolve, relative } from 'path'
+import { existsSync, readFileSync } from 'fs'
+import { BUILD_ID_FILE, CHUNK_GRAPH_MANIFEST } from 'next-server/constants'
 import {
   nextServer,
   nextBuild,
@@ -40,10 +40,26 @@ describe('Production Usage', () => {
 
   describe('File locations', () => {
     it('should build the app within the given `dist` directory', () => {
-      expect(existsSync(join(__dirname, `/../dist/${BUILD_ID_FILE}`))).toBeTruthy()
+      expect(
+        existsSync(join(__dirname, `/../dist/${BUILD_ID_FILE}`))
+      ).toBeTruthy()
     })
     it('should not build the app within the default `.next` directory', () => {
-      expect(existsSync(join(__dirname, `/../.next/${BUILD_ID_FILE}`))).toBeFalsy()
+      expect(
+        existsSync(join(__dirname, `/../.next/${BUILD_ID_FILE}`))
+      ).toBeFalsy()
+    })
+  })
+
+  describe('Module collection', () => {
+    it('should build a chunk graph file', () => {
+      const cgf = join(__dirname, `/../dist/${CHUNK_GRAPH_MANIFEST}`)
+      expect(existsSync(cgf)).toBeTruthy()
+      expect(
+        JSON.parse(readFileSync(cgf, 'utf8')).pages['/'].includes(
+          relative(appDir, resolve(__dirname, '..', 'pages', 'index.js'))
+        )
+      )
     })
   })
 })