Replace node.js url module with WHATWG URL (#14827)

Replace `url.parse` and `url.resolve` logic with whatwg `URL`, Bring in a customized `format` function to handle the node url objects that can be passed to router methods. This eliminates the need for `url` (and thus `native-url`) in core. Looks like it shaves off about 2.5Kb, according to the `size-limits` integration tests.

packages/next/client/link.tsx

 declare const __NEXT_DATA__: any
 
 import React, { Children } from 'react'
-import { parse, resolve, UrlObject } from 'url'
+import { UrlObject } from 'url'
 import { PrefetchOptions, NextRouter } from '../next-server/lib/router/router'
-import {
-  execOnce,
-  formatWithValidation,
-  getLocationOrigin,
-} from '../next-server/lib/utils'
+import { execOnce, getLocationOrigin } from '../next-server/lib/utils'
 import { useRouter } from './router'
-import { addBasePath } from '../next-server/lib/router/router'
-import { normalizeTrailingSlash } from './normalize-trailing-slash'
-
-function isLocal(href: string): boolean {
-  const url = parse(href, false, true)
-  const origin = parse(getLocationOrigin(), false, true)
-
-  return (
-    !url.host || (url.protocol === origin.protocol && url.host === origin.host)
-  )
+import { addBasePath, resolveHref } from '../next-server/lib/router/router'
+
+/**
+ * Detects whether a given url is from the same origin as the current page (browser only).
+ */
+function isLocal(url: string): boolean {
+  const locationOrigin = getLocationOrigin()
+  const resolved = new URL(url, locationOrigin)
+  return resolved.origin === locationOrigin
 }
 
 type Url = string | UrlObject
 
-function formatUrl(url: Url): string {
-  return (
-    url &&
-    formatWithValidation(
-      normalizeTrailingSlash(typeof url === 'object' ? url : parse(url))
-    )
-  )
-}
-
 export type LinkProps = {
   href: Url
   as?: Url
@@ -182,12 +168,10 @@ function Link(props: React.PropsWithChildren<LinkProps>) {
   const router = useRouter()
 
   const { href, as } = React.useMemo(() => {
-    const resolvedHref = resolve(router.pathname, formatUrl(props.href))
+    const resolvedHref = resolveHref(router.pathname, props.href)
     return {
       href: resolvedHref,
-      as: props.as
-        ? resolve(router.pathname, formatUrl(props.as))
-        : resolvedHref,
+      as: props.as ? resolveHref(router.pathname, props.as) : resolvedHref,
     }
   }, [router.pathname, props.href, props.as])
 

packages/next/client/normalize-trailing-slash.ts

 import { UrlObject } from 'url'
 
+/**
+ * Removes the trailing slash of a path if there is one. Preserves the root path `/`.
+ */
 export function removePathTrailingSlash(path: string): string {
   return path.endsWith('/') && path !== '/' ? path.slice(0, -1) : path
 }
 
+/**
+ * Normalizes the trailing slash of a path according to the `trailingSlash` option
+ * in `next.config.js`.
+ */
 const normalizePathTrailingSlash = process.env.__NEXT_TRAILING_SLASH
   ? (path: string): string => {
       if (/\.[^/]+\/?$/.test(path)) {
@@ -16,10 +23,17 @@ const normalizePathTrailingSlash = process.env.__NEXT_TRAILING_SLASH
     }
   : removePathTrailingSlash
 
-export function normalizeTrailingSlash(url: UrlObject): UrlObject {
+/**
+ * Normalizes the trailing slash of the path of a parsed url. Non-destructive.
+ */
+export function normalizeTrailingSlash(url: URL): URL
+export function normalizeTrailingSlash(url: UrlObject): UrlObject
+export function normalizeTrailingSlash(url: UrlObject | URL): UrlObject | URL {
   const normalizedPath =
     url.pathname && normalizePathTrailingSlash(url.pathname)
   return url.pathname === normalizedPath
     ? url
+    : url instanceof URL
+    ? Object.assign(new URL(url.href), { pathname: normalizedPath })
     : Object.assign({}, url, { pathname: normalizedPath })
 }

packages/next/client/page-loader.js

-import { parse } from 'url'
 import mitt from '../next-server/lib/mitt'
 import { isDynamicRoute } from './../next-server/lib/router/utils/is-dynamic'
 import { getRouteMatcher } from './../next-server/lib/router/utils/route-matcher'
 import { getRouteRegex } from './../next-server/lib/router/utils/route-regex'
+import { searchParamsToUrlQuery } from './../next-server/lib/router/utils/search-params-to-url-query'
+import { parseRelativeUrl } from './../next-server/lib/router/utils/parse-relative-url'
 import escapePathDelimiters from '../next-server/lib/router/utils/escape-path-delimiters'
 import getAssetPathFromRoute from './../next-server/lib/router/utils/get-asset-path-from-route'
 
@@ -111,8 +112,11 @@ export default class PageLoader {
    * @param {string} asPath the URL as shown in browser (virtual path); used for dynamic routes
    */
   getDataHref(href, asPath, ssg) {
-    const { pathname: hrefPathname, query, search } = parse(href, true)
-    const { pathname: asPathname } = parse(asPath)
+    const { pathname: hrefPathname, searchParams, search } = parseRelativeUrl(
+      href
+    )
+    const query = searchParamsToUrlQuery(searchParams)
+    const { pathname: asPathname } = parseRelativeUrl(asPath)
     const route = normalizeRoute(hrefPathname)
 
     const getHrefForSlug = (/** @type string */ path) => {
@@ -177,7 +181,7 @@ export default class PageLoader {
    * @param {string} asPath the URL as shown in browser (virtual path); used for dynamic routes
    */
   prefetchData(href, asPath) {
-    const { pathname: hrefPathname } = parse(href, true)
+    const { pathname: hrefPathname } = parseRelativeUrl(href)
     const route = normalizeRoute(hrefPathname)
     return this.promisedSsgManifest.then(
       (s, _dataHref) =>

packages/next/next-server/lib/router/router.ts

@@ -2,7 +2,7 @@
 // tslint:disable:no-console
 import { ParsedUrlQuery } from 'querystring'
 import { ComponentType } from 'react'
-import { parse, UrlObject } from 'url'
+import { UrlObject } from 'url'
 import mitt, { MittEmitter } from '../mitt'
 import {
   AppContextType,
@@ -15,6 +15,8 @@ import {
 import { isDynamicRoute } from './utils/is-dynamic'
 import { getRouteMatcher } from './utils/route-matcher'
 import { getRouteRegex } from './utils/route-regex'
+import { searchParamsToUrlQuery } from './utils/search-params-to-url-query'
+import { parseRelativeUrl } from './utils/parse-relative-url'
 import {
   normalizeTrailingSlash,
   removePathTrailingSlash,
@@ -36,20 +38,43 @@ function prepareRoute(path: string) {
 
 type Url = UrlObject | string
 
-function formatUrl(url: Url): string {
-  return url
-    ? formatWithValidation(
-        normalizeTrailingSlash(typeof url === 'object' ? url : parse(url))
-      )
-    : url
+/**
+ * Resolves a given hyperlink with a certain router state (basePath not included).
+ * Preserves absolute urls.
+ */
+export function resolveHref(currentPath: string, href: Url): string {
+  // we use a dummy base url for relative urls
+  const base = new URL(currentPath, 'http://n')
+  const urlAsString =
+    typeof href === 'string' ? href : formatWithValidation(href)
+  const finalUrl = normalizeTrailingSlash(new URL(urlAsString, base))
+  // if the origin didn't change, it means we received a relative href
+  return finalUrl.origin === base.origin
+    ? finalUrl.href.slice(finalUrl.origin.length)
+    : finalUrl.href
 }
 
-function prepareUrlAs(url: Url, as: Url) {
+function prepareUrlAs(router: NextRouter, url: Url, as: Url) {
   // If url and as provided as an object representation,
   // we'll format them into the string version here.
   return {
-    url: addBasePath(formatUrl(url)),
-    as: as ? addBasePath(formatUrl(as)) : as,
+    url: addBasePath(resolveHref(router.pathname, url)),
+    as: as ? addBasePath(resolveHref(router.pathname, as)) : as,
+  }
+}
+
+function tryParseRelativeUrl(
+  url: string
+): null | ReturnType<typeof parseRelativeUrl> {
+  try {
+    return parseRelativeUrl(url)
+  } catch (err) {
+    if (process.env.NODE_ENV !== 'production') {
+      throw new Error(
+        `Invalid href passed to router: ${url} https://err.sh/vercel/next.js/invalid-href-passed`
+      )
+    }
+    return null
   }
 }
 
@@ -318,14 +343,12 @@ export default class Router implements BaseRouter {
       return
     }
 
+    const { url, as, options } = e.state
+    const { pathname } = parseRelativeUrl(url)
+
     // Make sure we don't re-render on initial load,
     // can be caused by navigating back from an external site
-    if (
-      e.state &&
-      this.isSsr &&
-      e.state.as === this.asPath &&
-      parse(e.state.url).pathname === this.pathname
-    ) {
+    if (this.isSsr && as === this.asPath && pathname === this.pathname) {
       return
     }
 
@@ -335,7 +358,6 @@ export default class Router implements BaseRouter {
       return
     }
 
-    const { url, as, options } = e.state
     if (process.env.NODE_ENV !== 'production') {
       if (typeof url === 'undefined' || typeof as === 'undefined') {
         console.warn(
@@ -389,7 +411,7 @@ export default class Router implements BaseRouter {
    * @param options object you can define `shallow` and other options
    */
   push(url: Url, as: Url = url, options = {}) {
-    ;({ url, as } = prepareUrlAs(url, as))
+    ;({ url, as } = prepareUrlAs(this, url, as))
     return this.change('pushState', url, as, options)
   }
 
@@ -400,7 +422,7 @@ export default class Router implements BaseRouter {
    * @param options object you can define `shallow` and other options
    */
   replace(url: Url, as: Url = url, options = {}) {
-    ;({ url, as } = prepareUrlAs(url, as))
+    ;({ url, as } = prepareUrlAs(this, url, as))
     return this.change('replaceState', url, as, options)
   }
 
@@ -447,7 +469,12 @@ export default class Router implements BaseRouter {
         return resolve(true)
       }
 
-      let { pathname, query, protocol } = parse(url, true)
+      const parsed = tryParseRelativeUrl(url)
+
+      if (!parsed) return
+
+      let { pathname, searchParams } = parsed
+      const query = searchParamsToUrlQuery(searchParams)
 
       // url and as should always be prefixed with basePath by this
       // point by either next/link or router.push/replace so strip the
@@ -456,15 +483,6 @@ export default class Router implements BaseRouter {
         ? removePathTrailingSlash(delBasePath(pathname))
         : pathname
 
-      if (!pathname || protocol) {
-        if (process.env.NODE_ENV !== 'production') {
-          throw new Error(
-            `Invalid href passed to router: ${url} https://err.sh/vercel/next.js/invalid-href-passed`
-          )
-        }
-        return resolve(false)
-      }
-
       const cleanedAs = delBasePath(as)
 
       // If asked to change the current URL we should reload the current page
@@ -480,7 +498,7 @@ export default class Router implements BaseRouter {
       const { shallow = false } = options
 
       if (isDynamicRoute(route)) {
-        const { pathname: asPathname } = parse(cleanedAs)
+        const { pathname: asPathname } = parseRelativeUrl(cleanedAs)
         const routeRegex = getRouteRegex(route)
         const routeMatch = getRouteMatcher(routeRegex)(asPathname)
         if (!routeMatch) {
@@ -805,16 +823,11 @@ export default class Router implements BaseRouter {
     options: PrefetchOptions = {}
   ): Promise<void> {
     return new Promise((resolve, reject) => {
-      const { pathname, protocol } = parse(url)
+      const parsed = tryParseRelativeUrl(url)
 
-      if (!pathname || protocol) {
-        if (process.env.NODE_ENV !== 'production') {
-          throw new Error(
-            `Invalid href passed to router: ${url} https://err.sh/vercel/next.js/invalid-href-passed`
-          )
-        }
-        return
-      }
+      if (!parsed) return
+
+      const { pathname } = parsed
 
       // Prefetch is not supported in development mode because it would trigger on-demand-entries
       if (process.env.NODE_ENV !== 'production') {
@@ -873,7 +886,8 @@ export default class Router implements BaseRouter {
   }
 
   _getStaticData = (dataHref: string): Promise<object> => {
-    const pathname = prepareRoute(parse(dataHref).pathname!)
+    let { pathname } = parseRelativeUrl(dataHref)
+    pathname = prepareRoute(pathname)
 
     return process.env.NODE_ENV === 'production' && this.sdc[pathname]
       ? Promise.resolve(this.sdc[dataHref])

packages/next/next-server/lib/router/utils/format-url.ts

+// Format function modified from nodejs
+// Copyright Joyent, Inc. and other Node contributors.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a
+// copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to permit
+// persons to whom the Software is furnished to do so, subject to the
+// following conditions:
+//
+// The above copyright notice and this permission notice shall be included
+// in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+// USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+import { UrlObject } from 'url'
+import { encode as encodeQuerystring } from 'querystring'
+
+const slashedProtocols = /https?|ftp|gopher|file/
+
+export function formatUrl(urlObj: UrlObject) {
+  let { auth, hostname } = urlObj
+  let protocol = urlObj.protocol || ''
+  let pathname = urlObj.pathname || ''
+  let hash = urlObj.hash || ''
+  let query = urlObj.query || ''
+  let host: string | false = false
+
+  auth = auth ? encodeURIComponent(auth).replace(/%3A/i, ':') + '@' : ''
+
+  if (urlObj.host) {
+    host = auth + urlObj.host
+  } else if (hostname) {
+    host = auth + (~hostname.indexOf(':') ? `[${hostname}]` : hostname)
+    if (urlObj.port) {
+      host += ':' + urlObj.port
+    }
+  }
+
+  if (query && typeof query === 'object') {
+    // query = '' + new URLSearchParams(query);
+    query = encodeQuerystring(query)
+  }
+
+  let search = urlObj.search || (query && `?${query}`) || ''
+
+  if (protocol && protocol.substr(-1) !== ':') protocol += ':'
+
+  if (
+    urlObj.slashes ||
+    ((!protocol || slashedProtocols.test(protocol)) && host !== false)
+  ) {
+    host = '//' + (host || '')
+    if (pathname && pathname[0] !== '/') pathname = '/' + pathname
+  } else if (!host) {
+    host = ''
+  }
+
+  if (hash && hash[0] !== '#') hash = '#' + hash
+  if (search && search[0] !== '?') search = '?' + search
+
+  pathname = pathname.replace(/[?#]/g, encodeURIComponent)
+  search = search.replace('#', '%23')
+
+  return `${protocol}${host}${pathname}${search}${hash}`
+}

packages/next/next-server/lib/router/utils/parse-relative-url.ts

+const DUMMY_BASE = new URL('http://n')
+
+/**
+ * Parses path-relative urls (e.g. `/hello/world?foo=bar`). If url isn't path-relative
+ * (e.g. `./hello`) then at least base must be.
+ * Absolute urls are rejected.
+ */
+export function parseRelativeUrl(url: string, base?: string) {
+  const resolvedBase = base ? new URL(base, DUMMY_BASE) : DUMMY_BASE
+  const { pathname, searchParams, search, hash, href, origin } = new URL(
+    url,
+    resolvedBase
+  )
+  if (origin !== DUMMY_BASE.origin) {
+    throw new Error('invariant: invalid relative URL')
+  }
+  return {
+    pathname,
+    searchParams,
+    search,
+    hash,
+    href: href.slice(DUMMY_BASE.origin.length),
+  }
+}

packages/next/next-server/lib/router/utils/search-params-to-url-query.ts

+import { ParsedUrlQuery } from 'querystring'
+
+export function searchParamsToUrlQuery(
+  searchParams: URLSearchParams
+): ParsedUrlQuery {
+  const query: ParsedUrlQuery = {}
+  Array.from(searchParams.entries()).forEach(([key, value]) => {
+    if (typeof query[key] === 'undefined') {
+      query[key] = value
+    } else if (Array.isArray(query[key])) {
+      ;(query[key] as string[]).push(value)
+    } else {
+      query[key] = [query[key] as string, value]
+    }
+  })
+  return query
+}

packages/next/next-server/lib/utils.ts

 import { IncomingMessage, ServerResponse } from 'http'
 import { ParsedUrlQuery } from 'querystring'
 import { ComponentType } from 'react'
-import { format, URLFormatOptions, UrlObject } from 'url'
+import { UrlObject } from 'url'
+import { formatUrl } from './router/utils/format-url'
 import { ManifestItem } from '../server/load-components'
 import { NextRouter } from './router/router'
 import { Env } from '../../lib/load-env-config'
@@ -359,10 +360,7 @@ export const urlObjectKeys = [
   'slashes',
 ]
 
-export function formatWithValidation(
-  url: UrlObject,
-  options?: URLFormatOptions
-): string {
+export function formatWithValidation(url: UrlObject): string {
   if (process.env.NODE_ENV === 'development') {
     if (url !== null && typeof url === 'object') {
       Object.keys(url).forEach((key) => {
@@ -375,7 +373,7 @@ export function formatWithValidation(
     }
   }
 
-  return format(url as URL, options)
+  return formatUrl(url)
 }
 
 export const SP = typeof performance !== 'undefined'

test/integration/build-output/test/index.test.js

@@ -94,26 +94,31 @@ describe('Build Output', () => {
       expect(parseFloat(indexSize) - 265).toBeLessThanOrEqual(0)
       expect(indexSize.endsWith('B')).toBe(true)
 
-      // should be no bigger than 62 kb
-      expect(parseFloat(indexFirstLoad) - 61).toBeLessThanOrEqual(0)
+      // should be no bigger than 60 kb
+      expect(parseFloat(indexFirstLoad) - 60).toBeLessThanOrEqual(0)
       expect(indexFirstLoad.endsWith('kB')).toBe(true)
 
       expect(parseFloat(err404Size) - 3.4).toBeLessThanOrEqual(0)
       expect(err404Size.endsWith('kB')).toBe(true)
 
-      expect(parseFloat(err404FirstLoad) - 64).toBeLessThanOrEqual(0)
+      expect(parseFloat(err404FirstLoad) - 63).toBeLessThanOrEqual(0)
       expect(err404FirstLoad.endsWith('kB')).toBe(true)
 
-      expect(parseFloat(sharedByAll) - 61).toBeLessThanOrEqual(0)
+      expect(parseFloat(sharedByAll) - 59).toBeLessThanOrEqual(0)
       expect(sharedByAll.endsWith('kB')).toBe(true)
 
-      expect(parseFloat(_appSize) - 1000).toBeLessThanOrEqual(0)
-      expect(_appSize.endsWith('B')).toBe(true)
+      if (_appSize.endsWith('kB')) {
+        expect(parseFloat(_appSize)).toBe(1)
+        expect(_appSize.endsWith('kB')).toBe(true)
+      } else {
+        expect(parseFloat(_appSize) - 1000).toBeLessThanOrEqual(0)
+        expect(_appSize.endsWith(' B')).toBe(true)
+      }
 
-      expect(parseFloat(webpackSize) - 775).toBeLessThanOrEqual(0)
-      expect(webpackSize.endsWith('B')).toBe(true)
+      expect(parseFloat(webpackSize) - 752).toBeLessThanOrEqual(0)
+      expect(webpackSize.endsWith(' B')).toBe(true)
 
-      expect(parseFloat(mainSize) - 6.4).toBeLessThanOrEqual(0)
+      expect(parseFloat(mainSize) - 6.5).toBeLessThanOrEqual(0)
       expect(mainSize.endsWith('kB')).toBe(true)
 
       expect(parseFloat(frameworkSize) - 41).toBeLessThanOrEqual(0)

test/integration/size-limit/test/index.test.js

@@ -80,7 +80,7 @@ describe('Production response size', () => {
     )
 
     // These numbers are without gzip compression!
-    const delta = responseSizesBytes - 264 * 1024
+    const delta = responseSizesBytes - 261 * 1024
     expect(delta).toBeLessThanOrEqual(1024) // don't increase size more than 1kb
     expect(delta).toBeGreaterThanOrEqual(-1024) // don't decrease size more than 1kb without updating target
   })
@@ -100,7 +100,7 @@ describe('Production response size', () => {
     )
 
     // These numbers are without gzip compression!
-    const delta = responseSizesBytes - 169 * 1024
+    const delta = responseSizesBytes - 166 * 1024
     expect(delta).toBeLessThanOrEqual(1024) // don't increase size more than 1kb
     expect(delta).toBeGreaterThanOrEqual(-1024) // don't decrease size more than 1kb without updating target
   })