Add experimental image post-processing (#15875)

This PR adds a second experimental post-processing step for the framework introduced by @prateekbh in #14746. The image post-processing step scans the rendered document for the first few images and uses a simple heuristic to determine if the images should be automatically preloaded.

Analysis of quite a few production Next apps has shown that a lot of sites are taking a substantial hit to their [LCP](https://web.dev/lcp/) score because an image that's part of the "hero" element on the page is not preloaded and is getting downloaded with lower priority than the JavaScript bundles. This post-processor should automatically fix that for a lot of sites, without causing any real performance effects in cases where it fails to identify the hero image.

This feature is behind an experimental flag, and will be subject to quite a bit of experimentation and tweaking before it's ready to be made a default setting.

packages/next/build/webpack-config.ts

@@ -54,7 +54,6 @@ import WebpackConformancePlugin, {
   ReactSyncScriptsConformanceCheck,
 } from './webpack/plugins/webpack-conformance-plugin'
 import { WellKnownErrorsPlugin } from './webpack/plugins/wellknown-errors-plugin'
-
 type ExcludesFalse = <T>(x: T | false) => x is T
 
 const isWebpack5 = parseInt(webpack.version!) === 5
@@ -923,6 +922,9 @@ export default async function getBaseWebpackConfig(
         'process.env.__NEXT_OPTIMIZE_FONTS': JSON.stringify(
           config.experimental.optimizeFonts
         ),
+        'process.env.__NEXT_OPTIMIZE_IMAGES': JSON.stringify(
+          config.experimental.optimizeImages
+        ),
         'process.env.__NEXT_SCROLL_RESTORATION': JSON.stringify(
           config.experimental.scrollRestoration
         ),

packages/next/export/index.ts

@@ -407,6 +407,7 @@ export default async function exportApp(
         buildExport: options.buildExport,
         serverless: isTargetLikeServerless(nextConfig.target),
         optimizeFonts: nextConfig.experimental.optimizeFonts,
+        optimizeImages: nextConfig.experimental.optimizeImages,
       })
 
       for (const validation of result.ampValidations || []) {

packages/next/export/worker.ts

@@ -47,6 +47,7 @@ interface ExportPageInput {
   subFolders: string
   serverless: boolean
   optimizeFonts: boolean
+  optimizeImages: boolean
 }
 
 interface ExportPageResults {
@@ -64,6 +65,7 @@ interface RenderOpts {
   hybridAmp?: boolean
   inAmpMode?: boolean
   optimizeFonts?: boolean
+  optimizeImages?: boolean
   fontManifest?: FontManifest
 }
 
@@ -84,6 +86,7 @@ export default async function exportPage({
   subFolders,
   serverless,
   optimizeFonts,
+  optimizeImages,
 }: ExportPageInput): Promise<ExportPageResults> {
   let results: ExportPageResults = {
     ampValidations: [],
@@ -221,6 +224,8 @@ export default async function exportPage({
             ampPath,
             /// @ts-ignore
             optimizeFonts,
+            /// @ts-ignore
+            optimizeImages,
             fontManifest: optimizeFonts
               ? requireFontManifest(distDir, serverless)
               : null,
@@ -268,12 +273,16 @@ export default async function exportPage({
         if (optimizeFonts) {
           process.env.__NEXT_OPTIMIZE_FONTS = JSON.stringify(true)
         }
+        if (optimizeImages) {
+          process.env.__NEXT_OPTIMIZE_IMAGES = JSON.stringify(true)
+        }
         curRenderOpts = {
           ...components,
           ...renderOpts,
           ampPath,
           params,
           optimizeFonts,
+          optimizeImages,
           fontManifest: optimizeFonts
             ? requireFontManifest(distDir, serverless)
             : null,

packages/next/next-server/lib/post-process.ts

@@ -2,9 +2,12 @@ import { parse, HTMLElement } from 'node-html-parser'
 import { OPTIMIZED_FONT_PROVIDERS } from './constants'
 
 const MIDDLEWARE_TIME_BUDGET = 10
+const MAXIMUM_IMAGE_PRELOADS = 2
+const IMAGE_PRELOAD_SIZE_THRESHOLD = 2500
 
 type postProcessOptions = {
   optimizeFonts: boolean
+  optimizeImages: boolean
 }
 
 type renderOptions = {
@@ -149,6 +152,83 @@ class FontOptimizerMiddleware implements PostProcessMiddleware {
   }
 }
 
+class ImageOptimizerMiddleware implements PostProcessMiddleware {
+  inspect(originalDom: HTMLElement, _data: postProcessData) {
+    const imgElements = originalDom.querySelectorAll('img')
+    let eligibleImages: Array<HTMLElement> = []
+    for (let i = 0; i < imgElements.length; i++) {
+      if (isImgEligible(imgElements[i])) {
+        eligibleImages.push(imgElements[i])
+      }
+      if (eligibleImages.length >= MAXIMUM_IMAGE_PRELOADS) {
+        break
+      }
+    }
+    _data.preloads.images = eligibleImages.map((el) => el.getAttribute('src'))
+  }
+  mutate = async (markup: string, _data: postProcessData) => {
+    let result = markup
+    let imagePreloadTags = _data.preloads.images
+      .filter((imgHref) => !preloadTagAlreadyExists(markup, imgHref))
+      .reduce(
+        (acc, imgHref) => acc + `<link rel="preload" href="${imgHref}"/>`,
+        ''
+      )
+    return result.replace(
+      /<link rel="preload"/,
+      `${imagePreloadTags}<link rel="preload"`
+    )
+  }
+}
+
+function isImgEligible(imgElement: HTMLElement): boolean {
+  return (
+    imgElement.hasAttribute('src') &&
+    imageIsNotTooSmall(imgElement) &&
+    imageIsNotHidden(imgElement)
+  )
+}
+
+function preloadTagAlreadyExists(html: string, href: string) {
+  const regex = new RegExp(`<link[^>]*href[^>]*${href}`)
+  return html.match(regex)
+}
+
+function imageIsNotTooSmall(imgElement: HTMLElement): boolean {
+  // Skip images without both height and width--we don't know enough to say if
+  // they are too small
+  if (
+    !(imgElement.hasAttribute('height') && imgElement.hasAttribute('width'))
+  ) {
+    return true
+  }
+  try {
+    if (
+      parseInt(imgElement.getAttribute('height')) *
+        parseInt(imgElement.getAttribute('width')) <=
+      IMAGE_PRELOAD_SIZE_THRESHOLD
+    ) {
+      return false
+    }
+  } catch (err) {
+    return true
+  }
+  return true
+}
+
+// Traverse up the dom from each image to see if it or any of it's
+// ancestors have the hidden attribute.
+function imageIsNotHidden(imgElement: HTMLElement): boolean {
+  let activeElement = imgElement
+  while (activeElement.parentNode) {
+    if (activeElement.hasAttribute('hidden')) {
+      return false
+    }
+    activeElement = activeElement.parentNode as HTMLElement
+  }
+  return true
+}
+
 // Initialization
 registerPostProcessor(
   'Inline-Fonts',
@@ -158,4 +238,11 @@ registerPostProcessor(
   (options) => options.optimizeFonts || process.env.__NEXT_OPTIMIZE_FONTS
 )
 
+registerPostProcessor(
+  'Preload Images',
+  new ImageOptimizerMiddleware(),
+  // @ts-ignore
+  (options) => options.optimizeImages || process.env.__NEXT_OPTIMIZE_IMAGES
+)
+
 export default processHTML

packages/next/next-server/server/config.ts

@@ -52,6 +52,7 @@ const defaultConfig: { [key: string]: any } = {
     pageEnv: false,
     productionBrowserSourceMaps: false,
     optimizeFonts: false,
+    optimizeImages: false,
     scrollRestoration: false,
   },
   future: {

packages/next/next-server/server/next-server.ts

@@ -122,6 +122,7 @@ export default class Server {
     basePath: string
     optimizeFonts: boolean
     fontManifest: FontManifest
+    optimizeImages: boolean
   }
   private compression?: Middleware
   private onErrorMiddleware?: ({ err }: { err: Error }) => Promise<void>
@@ -172,6 +173,7 @@ export default class Server {
       fontManifest: this.nextConfig.experimental.optimizeFonts
         ? requireFontManifest(this.distDir, this._isLikeServerless)
         : null,
+      optimizeImages: this.nextConfig.experimental.optimizeImages,
     }
 
     // Only the `publicRuntimeConfig` key is exposed to the client side
@@ -236,6 +238,9 @@ export default class Server {
     if (this.renderOpts.optimizeFonts) {
       process.env.__NEXT_OPTIMIZE_FONTS = JSON.stringify(true)
     }
+    if (this.renderOpts.optimizeImages) {
+      process.env.__NEXT_OPTIMIZE_IMAGES = JSON.stringify(true)
+    }
   }
 
   protected currentPhase(): string {

packages/next/next-server/server/render.tsx

@@ -147,6 +147,7 @@ export type RenderOptsPartial = {
   unstable_runtimeJS?: false
   optimizeFonts: boolean
   fontManifest?: FontManifest
+  optimizeImages: boolean
   devOnlyCacheBusterQueryString?: string
 }
 
@@ -813,6 +814,7 @@ export async function renderToHTML(
     },
     {
       optimizeFonts: renderOpts.optimizeFonts,
+      optimizeImages: renderOpts.optimizeImages,
     }
   )
 

test/integration/font-optimization/next.config.js

+module.exports = { target: 'serverless', experimental: { optimizeFonts: true } }

test/integration/image-optimization/next.config.js

+module.exports = {
+  target: 'serverless',
+  experimental: { optimizeImages: true },
+}

test/integration/image-optimization/pages/index.js

+import React from 'react'
+
+const Page = () => {
+  return (
+    <div>
+      <link rel="preload" href="already-preloaded.jpg" />
+      <img src="already-preloaded.jpg" />
+      <img src="tiny-image.jpg" width="20" height="20" />
+      <img src="hidden-image-1.jpg" hidden />
+      <div hidden>
+        <img src="hidden-image-2.jpg" />
+      </div>
+      <img src="main-image-1.jpg" />
+      <div>
+        <img src="main-image-2.jpg" />
+      </div>
+      <img src="main-image-3.jpg" />
+      <img src="main-image-4.jpg" />
+      <img src="main-image-5.jpg" />
+    </div>
+  )
+}
+
+export default Page

test/integration/image-optimization/pages/stars.js

+function Home({ stars }) {
+  return (
+    <div className="container">
+      <main>
+        <div>
+          <link rel="preload" href="already-preloaded.jpg" />
+          <img src="already-preloaded.jpg" />
+          <img src="tiny-image.jpg" width="20" height="20" />
+          <img src="hidden-image-1.jpg" hidden />
+          <div hidden>
+            <img src="hidden-image-2.jpg" />
+          </div>
+          <img src="main-image-1.jpg" />
+          <img src="main-image-2.jpg" />
+          <img src="main-image-3.jpg" />
+          <img src="main-image-4.jpg" />
+          <img src="main-image-5.jpg" />
+        </div>
+        <div>Next stars: {stars}</div>
+      </main>
+    </div>
+  )
+}
+
+Home.getInitialProps = async () => {
+  return { stars: Math.random() * 1000 }
+}
+
+export default Home

test/integration/image-optimization/pages/static-head.js

+import React from 'react'
+import Head from 'next/head'
+
+const Page = () => {
+  return (
+    <>
+      <Head>
+        <link
+          href="https://fonts.googleapis.com/css2?family=Modak"
+          rel="stylesheet"
+        />
+      </Head>
+      <div>Hi!</div>
+    </>
+  )
+}
+
+export default Page

test/integration/image-optimization/test/index.test.js

+/* eslint-env jest */
+
+import { join } from 'path'
+import {
+  killApp,
+  findPort,
+  nextStart,
+  nextBuild,
+  renderViaHTTP,
+} from 'next-test-utils'
+import fs from 'fs-extra'
+
+jest.setTimeout(1000 * 30)
+
+const appDir = join(__dirname, '../')
+const nextConfig = join(appDir, 'next.config.js')
+let appPort
+let app
+
+function runTests() {
+  describe('On a static page', () => {
+    checkImagesOnPage('/')
+  })
+
+  describe('On an SSR page', () => {
+    checkImagesOnPage('/stars')
+  })
+}
+
+function checkImagesOnPage(path) {
+  it('should not preload tiny images', async () => {
+    const html = await renderViaHTTP(appPort, path)
+    expect(html).not.toContain('<link rel="preload" href="tiny-image.jpg"/>')
+  })
+  it('should not add a preload if one already exists', async () => {
+    let html = await renderViaHTTP(appPort, path)
+    html = html.replace(
+      '<link rel="preload" href="already-preloaded.jpg"/>',
+      ''
+    )
+    expect(html).not.toContain(
+      '<link rel="preload" href="already-preloaded.jpg"/>'
+    )
+  })
+  it('should not preload hidden images', async () => {
+    const html = await renderViaHTTP(appPort, path)
+    expect(html).not.toContain(
+      '<link rel="preload" href="hidden-image-1.jpg"/>'
+    )
+    expect(html).not.toContain(
+      '<link rel="preload" href="hidden-image-2.jpg"/>'
+    )
+  })
+  it('should preload exactly two eligible images', async () => {
+    const html = await renderViaHTTP(appPort, path)
+    expect(html).toContain('<link rel="preload" href="main-image-1.jpg"/>')
+    expect(html).not.toContain('<link rel="preload" href="main-image-2.jpg"/>')
+  })
+}
+
+describe('Image optimization for SSR apps', () => {
+  beforeAll(async () => {
+    await fs.writeFile(
+      nextConfig,
+      `module.exports = { experimental: {optimizeImages: true} }`,
+      'utf8'
+    )
+    await nextBuild(appDir)
+    appPort = await findPort()
+    app = await nextStart(appDir, appPort)
+  })
+  afterAll(() => killApp(app))
+  runTests()
+})
+
+describe('Image optimization for serverless apps', () => {
+  beforeAll(async () => {
+    await fs.writeFile(
+      nextConfig,
+      `module.exports = { target: 'serverless', experimental: {optimizeImages: true} }`,
+      'utf8'
+    )
+    await nextBuild(appDir)
+    appPort = await findPort()
+    app = await nextStart(appDir, appPort)
+  })
+  afterAll(() => killApp(app))
+  runTests()
+})

yarn.lock

@@ -7784,6 +7784,11 @@ he@1.1.1:
   resolved "https://registry.yarnpkg.com/he/-/he-1.1.1.tgz#93410fd21b009735151f8868c2f271f3427e23fd"
   integrity sha1-k0EP0hsAlzUVH4howvJx80J+I/0=
 
+he@1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/he/-/he-1.1.1.tgz#93410fd21b009735151f8868c2f271f3427e23fd"
+  integrity sha1-k0EP0hsAlzUVH4howvJx80J+I/0=
+
 header-case@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/header-case/-/header-case-1.0.1.tgz#9535973197c144b09613cd65d317ef19963bd02d"