Revise dynamic route generation (mark 3) (#7473)

* Revise dynamic route generation
This implements a new tree-based route sorting algorithm that uses a Depth-First-Traversal approach to correctly sort the routes.

This provides better clarity over a `.sort()` based approach and will scale well as we add new features in the future.

* Update import

packages/next-server/lib/router/router.ts

@@ -13,7 +13,7 @@ import {
   NextPageContext,
 } from '../utils'
 import { rewriteUrlForNextExport } from './rewrite-url-for-export'
-import { getRouteRegex } from './utils'
+import { getRouteRegex } from './utils/route-regex'
 
 function toRoute(path: string): string {
   return path.replace(/\/$/, '') || '/'

packages/next-server/lib/router/utils/route-matcher.ts

+import { getRouteRegex } from './route-regex'
+
+export function getRouteMatcher(routeRegex: ReturnType<typeof getRouteRegex>) {
+  const { re, groups } = routeRegex
+  return (pathname: string | undefined) => {
+    const routeMatch = re.exec(pathname!)
+    if (!routeMatch) {
+      return false
+    }
+
+    const params: { [paramName: string]: string } = {}
+
+    Object.keys(groups).forEach((slugName: string) => {
+      const m = routeMatch[groups[slugName]]
+      if (m !== undefined) {
+        params[slugName] = decodeURIComponent(m)
+      }
+    })
+    return params
+  }
+}

packages/next-server/lib/router/utils.ts → packages/next-server/lib/router/utils/route-regex.ts

 export function getRouteRegex(
-  route: string
+  normalizedRoute: string
 ): { re: RegExp; groups: { [groupName: string]: number } } {
-  const escapedRoute = (route.replace(/\/$/, '') || '/').replace(
+  const escapedRoute = (normalizedRoute.replace(/\/$/, '') || '/').replace(
     /[|\\{}()[\]^$+*?.-]/g,
     '\\$&'
   )
@@ -33,23 +33,3 @@ export function getRouteRegex(
     groups,
   }
 }
-
-export function getRouteMatch(route: string) {
-  const { re, groups } = getRouteRegex(route)
-  return (pathname: string | undefined) => {
-    const routeMatch = re.exec(pathname!)
-    if (!routeMatch) {
-      return false
-    }
-
-    const params: { [paramName: string]: string } = {}
-
-    Object.keys(groups).forEach((slugName: string) => {
-      const m = routeMatch[groups[slugName]]
-      if (m !== undefined) {
-        params[slugName] = decodeURIComponent(m)
-      }
-    })
-    return params
-  }
-}

packages/next-server/lib/router/utils/sorted-routes.ts

+class UrlNode {
+  placeholder: boolean = true
+  children: Map<string, UrlNode> = new Map()
+  slugName: string | null = null
+
+  get hasSlug() {
+    return this.slugName != null
+  }
+
+  insert(urlPath: string): void {
+    this._insert(urlPath.split('/').filter(Boolean))
+  }
+
+  smoosh(): string[] {
+    return this._smoosh()
+  }
+
+  private _smoosh(prefix: string = '/'): string[] {
+    const childrenPaths = [...this.children.keys()].sort()
+    if (this.hasSlug) {
+      childrenPaths.splice(childrenPaths.indexOf('$'), 1)
+    }
+
+    const routes = childrenPaths
+      .map(c => this.children.get(c)!._smoosh(`${prefix}${c}/`))
+      .reduce((prev, curr) => [...prev, ...curr], [])
+
+    if (this.hasSlug) {
+      routes.push(
+        ...this.children.get('$')!._smoosh(`${prefix}$${this.slugName}/`)
+      )
+    }
+
+    if (!this.placeholder) {
+      routes.unshift(prefix === '/' ? '/' : prefix.slice(0, -1))
+    }
+
+    return routes
+  }
+
+  private _insert(urlPaths: string[]): void {
+    if (urlPaths.length === 0) {
+      this.placeholder = false
+      return
+    }
+
+    let [nextSegment] = urlPaths
+    if (nextSegment.startsWith('$')) {
+      const slugName = nextSegment.substring(1)
+      if (this.hasSlug && slugName !== this.slugName) {
+        throw new Error(
+          'You cannot use different slug names for the same dynamic path.'
+        )
+      }
+
+      this.slugName = slugName
+      nextSegment = '$'
+    }
+
+    if (!this.children.has(nextSegment)) {
+      this.children.set(nextSegment, new UrlNode())
+    }
+
+    this.children.get(nextSegment)!._insert(urlPaths.slice(1))
+  }
+}
+
+export function getSortedRoutes(normalizedPages: string[]): string[] {
+  const root = new UrlNode()
+  normalizedPages.forEach(page => root.insert(page))
+  return root.smoosh()
+}

packages/next-server/server/next-server.ts

-/* eslint-disable import/first */
 import fs from 'fs'
 import { IncomingMessage, ServerResponse } from 'http'
 import { join, resolve, sep } from 'path'
@@ -15,7 +14,9 @@ import {
   PHASE_PRODUCTION_SERVER,
   SERVER_DIRECTORY,
 } from '../lib/constants'
-import { getRouteMatch } from '../lib/router/utils'
+import { getRouteMatcher } from '../lib/router/utils/route-matcher'
+import { getRouteRegex } from '../lib/router/utils/route-regex'
+import { getSortedRoutes } from '../lib/router/utils/sorted-routes'
 import * as envConfig from '../lib/runtime-config'
 import loadConfig from './config'
 import { recursiveReadDirSync } from './lib/recursive-readdir-sync'
@@ -310,14 +311,10 @@ export default class Server {
     const dynamicRoutedPages = Object.keys(manifest.pages).filter(p =>
       p.includes('/$')
     )
-    return dynamicRoutedPages
-      .map(page => ({
-        page,
-        match: getRouteMatch(page),
-      }))
-      .sort((a, b) =>
-        Math.sign(a.page.match(/\/\$/g)!.length - b.page.match(/\/\$/g)!.length)
-      )
+    return getSortedRoutes(dynamicRoutedPages).map(page => ({
+      page,
+      match: getRouteMatcher(getRouteRegex(page)),
+    }))
   }
 
   private async run(

packages/next/server/next-dev-server.js

@@ -9,7 +9,9 @@ import { ampValidation } from '../build/output/index'
 import * as Log from '../build/output/log'
 import { verifyTypeScriptSetup } from '../lib/verifyTypeScriptSetup'
 import Watchpack from 'watchpack'
-import { getRouteMatch } from 'next-server/dist/lib/router/utils'
+import { getRouteMatcher } from 'next-server/dist/lib/router/utils/route-matcher'
+import { getRouteRegex } from 'next-server/dist/lib/router/utils/route-regex'
+import { getSortedRoutes } from 'next-server/dist/lib/router/utils/sorted-routes'
 
 const React = require('react')
 
@@ -102,7 +104,7 @@ export default class DevServer extends Server {
       wp.watch([], [pagesDir], 0)
 
       wp.on('aggregated', () => {
-        const newDynamicRoutes = []
+        const dynamicRoutedPages = []
         const knownFiles = wp.getTimeInfoEntries()
         for (const [fileName, { accuracy }] of knownFiles) {
           if (accuracy === undefined) {
@@ -118,15 +120,14 @@ export default class DevServer extends Server {
           pageName = pageName.slice(0, -pageExt.length)
 
           pageName = pageName.replace(/\/index$/, '')
-          newDynamicRoutes.push({
-            page: pageName,
-            match: getRouteMatch(pageName)
-          })
+          dynamicRoutedPages.push(pageName)
         }
 
-        this.dynamicRoutes = newDynamicRoutes.sort((a, b) =>
-          Math.sign(a.page.match(/\/\$/g).length - b.page.match(/\/\$/g).length)
-        )
+        this.dynamicRoutes = getSortedRoutes(dynamicRoutedPages).map(page => ({
+          page,
+          match: getRouteMatcher(getRouteRegex(page))
+        }))
+
         resolve()
       })
     })

test/unit/__snapshots__/page-route-sorter.test.js.snap

+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`getSortedRoutes correctly sorts required slugs 1`] = `
+Array [
+  "/",
+  "/apples/$ab/$cd/ef",
+  "/blog/$id",
+  "/blog/$id/comments/$cid",
+  "/foo/$d/bar/baz/$f",
+  "/posts",
+  "/posts/$id",
+  "/$root-slug",
+]
+`;

test/unit/page-route-sorter.test.js

+/* eslint-env jest */
+import { getSortedRoutes } from 'next-server/dist/lib/router/utils/sorted-routes'
+
+describe('getSortedRoutes', () => {
+  it('does not add extra routes', () => {
+    expect(getSortedRoutes(['/posts'])).toEqual(['/posts'])
+
+    expect(getSortedRoutes(['/posts/$id'])).toEqual(['/posts/$id'])
+    expect(getSortedRoutes(['/posts/$id/foo'])).toEqual(['/posts/$id/foo'])
+
+    expect(getSortedRoutes(['/posts/$id/$foo/bar'])).toEqual([
+      '/posts/$id/$foo/bar'
+    ])
+    expect(getSortedRoutes(['/posts/$id/baz/$foo/bar'])).toEqual([
+      '/posts/$id/baz/$foo/bar'
+    ])
+  })
+
+  it('correctly sorts required slugs', () => {
+    expect(
+      getSortedRoutes([
+        '/posts',
+        '/$root-slug',
+        '/',
+        '/posts/$id',
+        '/blog/$id/comments/$cid',
+        '/blog/$id',
+        '/foo/$d/bar/baz/$f',
+        '/apples/$ab/$cd/ef'
+      ])
+    ).toMatchSnapshot()
+  })
+})