First pass of runtime amp validator (#6708)

packages/next/build/output/index.ts

+import chalk from 'chalk'
+import textTable from 'next/dist/compiled/text-table'
 import createStore from 'next/dist/compiled/unistore'
+import stripAnsi from 'strip-ansi'
 
-import { store, OutputState } from './store'
 import formatWebpackMessages from '../../client/dev-error-overlay/format-webpack-messages'
+import { OutputState, store as consoleStore } from './store'
 
 export function startedDevelopmentServer(appUrl: string) {
-  store.setState({ appUrl })
+  consoleStore.setState({ appUrl })
 }
 
 let previousClient: any = null
@@ -18,9 +21,21 @@ type WebpackStatus =
       warnings: string[] | null
     }
 
-type WebpackStatusStore = {
+type AmpStatus = {
+  message: string
+  line: number
+  col: number
+  specUrl: string | null
+}
+
+type AmpPageStatus = {
+  [page: string]: { errors: AmpStatus[]; warnings: AmpStatus[] }
+}
+
+type BuildStatusStore = {
   client: WebpackStatus
   server: WebpackStatus
+  amp: AmpPageStatus
 }
 
 enum WebpackStatusPhase {
@@ -43,35 +58,116 @@ function getWebpackStatusPhase(status: WebpackStatus): WebpackStatusPhase {
   return WebpackStatusPhase.COMPILED
 }
 
-const webpackStore = createStore<WebpackStatusStore>()
+function formatAmpMessages(amp: AmpPageStatus) {
+  let output = chalk.bold('Amp Validation') + '\n\n'
+  let messages: string[][] = []
+
+  const chalkError = chalk.red('error')
+  function ampError(page: string, error: AmpStatus) {
+    messages.push([page, chalkError, error.message, error.specUrl || ''])
+  }
+
+  const chalkWarn = chalk.yellow('warn')
+  function ampWarn(page: string, warn: AmpStatus) {
+    messages.push([page, chalkWarn, warn.message, warn.specUrl || ''])
+  }
+
+  for (const page in amp) {
+    const { errors, warnings } = amp[page]
+    if (errors.length) {
+      ampError(page, errors[0])
+      for (let index = 1; index < errors.length; ++index) {
+        ampError('', errors[index])
+      }
+    }
+    if (warnings.length) {
+      ampWarn(errors.length ? '' : page, warnings[0])
+      for (let index = 1; index < warnings.length; ++index) {
+        ampWarn('', warnings[index])
+      }
+    }
+    messages.push(['', '', '', ''])
+  }
+
+  output += textTable(messages, {
+    align: ['l', 'l', 'l', 'l'],
+    stringLength(str: string) {
+      return stripAnsi(str).length
+    },
+  })
+
+  return output
+}
+
+const buildStore = createStore<BuildStatusStore>()
 
-webpackStore.subscribe(state => {
-  const { client, server } = state
+buildStore.subscribe(state => {
+  const { amp, client, server } = state
 
   const [{ status }] = [
     { status: client, phase: getWebpackStatusPhase(client) },
     { status: server, phase: getWebpackStatusPhase(server) },
   ].sort((a, b) => a.phase.valueOf() - b.phase.valueOf())
 
-  const { bootstrap: bootstrapping, appUrl } = store.getState()
+  const { bootstrap: bootstrapping, appUrl } = consoleStore.getState()
   if (bootstrapping && status.loading) {
     return
   }
 
-  let nextStoreState: OutputState = {
+  let partialState: Partial<OutputState> = {
     bootstrap: false,
     appUrl: appUrl!,
-    ...status,
   }
-  store.setState(nextStoreState, true)
+
+  if (status.loading) {
+    consoleStore.setState(
+      { ...partialState, loading: true } as OutputState,
+      true
+    )
+  } else {
+    let { errors, warnings } = status
+
+    if (errors == null && Object.keys(amp).length > 0) {
+      warnings = (warnings || []).concat(formatAmpMessages(amp))
+    }
+
+    consoleStore.setState(
+      { ...partialState, loading: false, errors, warnings } as OutputState,
+      true
+    )
+  }
 })
 
+export function ampValidation(
+  page: string,
+  errors: AmpStatus[],
+  warnings: AmpStatus[]
+) {
+  const { amp } = buildStore.getState()
+  if (!(errors.length || warnings.length)) {
+    buildStore.setState({
+      amp: Object.keys(amp)
+        .filter(k => k !== page)
+        .sort()
+        .reduce((a, c) => ((a[c] = amp[c]), a), {} as any),
+    })
+    return
+  }
+
+  const newAmp: AmpPageStatus = { ...amp, [page]: { errors, warnings } }
+  buildStore.setState({
+    amp: Object.keys(newAmp)
+      .sort()
+      .reduce((a, c) => ((a[c] = newAmp[c]), a), {} as any),
+  })
+}
+
 export function watchCompiler(client: any, server: any) {
   if (previousClient === client && previousServer === server) {
     return
   }
 
-  webpackStore.setState({
+  buildStore.setState({
     client: { loading: true },
     server: { loading: true },
   })
@@ -86,6 +182,8 @@ export function watchCompiler(client: any, server: any) {
     })
 
     compiler.hooks.done.tap(`NextJsDone-${key}`, (stats: any) => {
+      buildStore.setState({ amp: {} })
+
       const { errors, warnings } = formatWebpackMessages(
         stats.toJson({ all: false, warnings: true, errors: true })
       )
@@ -99,10 +197,10 @@ export function watchCompiler(client: any, server: any) {
   }
 
   tapCompiler('client', client, status =>
-    webpackStore.setState({ client: status })
+    buildStore.setState({ client: status })
   )
   tapCompiler('server', server, status =>
-    webpackStore.setState({ server: status })
+    buildStore.setState({ server: status })
   )
 
   previousClient = client

packages/next/package.json

@@ -50,6 +50,7 @@
     "@babel/runtime": "7.1.2",
     "@babel/runtime-corejs2": "7.1.2",
     "@babel/template": "7.1.2",
+    "amphtml-validator": "1.0.23",
     "arg": "3.0.0",
     "async-sema": "2.2.0",
     "autodll-webpack-plugin": "0.4.2",
@@ -98,6 +99,7 @@
     "@taskr/clear": "1.1.0",
     "@taskr/esnext": "1.1.0",
     "@taskr/watch": "1.1.0",
+    "@types/amphtml-validator": "1.0.0",
     "@types/babel-types": "7.0.4",
     "@types/babel__core": "7.0.4",
     "@types/babel__generator": "7.0.1",
@@ -111,9 +113,12 @@
     "@types/nanoid": "1.2.0",
     "@types/node-fetch": "2.1.4",
     "@types/resolve": "0.0.8",
+    "@types/strip-ansi": "3.0.0",
+    "@types/text-table": "0.2.0",
     "@types/webpack-sources": "0.1.5",
     "@zeit/ncc": "0.15.2",
     "taskr": "1.1.0",
+    "text-table": "0.2.0",
     "typescript": "3.1.6",
     "unistore": "3.2.1"
   },

packages/next/pages/_document.js

@@ -178,13 +178,13 @@ export class Head extends Component {
       if (!child) return child
       const { type, props } = child
       let badProp
-      
+
       if (type === 'meta' && props.name === 'viewport') {
         badProp = 'name="viewport"'
       } else if (type === 'link' && props.rel === 'canonical') {
         badProp = 'rel="canonical"'
       }
-      
+
       if (badProp) {
         console.warn(`Found conflicting amp tag "${child.type}" with conflicting prop ${badProp}. https://err.sh/next.js/conflicting-amp-tag`)
         return null
@@ -387,6 +387,7 @@ export class NextScript extends Component {
                   this.context._documentProps
                 ),
               }}
+              data-amp-development-mode-only
             />
           )}
           {devFiles
@@ -396,6 +397,7 @@ export class NextScript extends Component {
                   src={`${assetPrefix}/_next/${file}${_devOnlyInvalidateCacheQueryString}`}
                   nonce={this.props.nonce}
                   crossOrigin={this.props.crossOrigin || process.crossOrigin}
+                  data-amp-development-mode-only
                 />
               ))
             : null}

packages/next/server/next-dev-server.js

@@ -4,6 +4,8 @@ import HotReloader from './hot-reloader'
 import { route } from 'next-server/dist/server/router'
 import { PHASE_DEVELOPMENT_SERVER } from 'next-server/constants'
 import ErrorDebug from './error-debug'
+import AmpHtmlValidator from 'amphtml-validator'
+import { ampValidation } from '../build/output/index'
 
 export default class DevServer extends Server {
   constructor (options) {
@@ -91,6 +93,27 @@ export default class DevServer extends Server {
     return routes
   }
 
+  _filterAmpDevelopmentScript (html, event) {
+    if (event.code !== 'DISALLOWED_SCRIPT_TAG') {
+      return true
+    }
+
+    const snippetChunks = html.split('\n')
+
+    let snippet
+    if (
+      !(snippet = html.split('\n')[event.line - 1]) ||
+      !(snippet = snippet.substring(event.col))
+    ) {
+      return true
+    }
+
+    snippet = snippet + snippetChunks.slice(event.line).join('\n')
+    snippet = snippet.substring(0, snippet.indexOf('</script>'))
+
+    return !snippet.includes('data-amp-development-mode-only')
+  }
+
   async renderToHTML (req, res, pathname, query, options) {
     const compilationErr = await this.getCompilationError(pathname)
     if (compilationErr) {
@@ -108,7 +131,20 @@ export default class DevServer extends Server {
       }
       if (!this.quiet) console.error(err)
     }
-    return super.renderToHTML(req, res, pathname, query, options)
+    const html = await super.renderToHTML(req, res, pathname, query, options)
+    if (options.amphtml && pathname !== '/_error') {
+      await AmpHtmlValidator.getInstance().then(validator => {
+        const result = validator.validateString(html)
+        ampValidation(
+          pathname,
+          result.errors
+            .filter(e => e.severity === 'ERROR')
+            .filter(e => this._filterAmpDevelopmentScript(html, e)),
+          result.errors.filter(e => e.severity !== 'ERROR')
+        )
+      })
+    }
+    return html
   }
 
   async renderErrorToHTML (err, req, res, pathname, query) {
@@ -121,8 +157,9 @@ export default class DevServer extends Server {
     }
 
     if (!err && res.statusCode === 500) {
-      err = new Error('An undefined error was thrown sometime during render... ' +
-       'See https://err.sh/zeit/next.js/threw-undefined'
+      err = new Error(
+        'An undefined error was thrown sometime during render... ' +
+          'See https://err.sh/zeit/next.js/threw-undefined'
       )
     }
 

packages/next/taskfile-ncc.js

 'use strict'
 
 const ncc = require('@zeit/ncc')
-const { existsSync, copyFileSync } = require('fs')
+const { existsSync, readFileSync } = require('fs')
 const { basename, dirname, extname, join, relative } = require('path')
 
 module.exports = function (task) {
@@ -47,7 +47,11 @@ function writePackageManifest (packageName) {
 
   const potentialLicensePath = join(dirname(packagePath), './LICENSE')
   if (existsSync(potentialLicensePath)) {
-    copyFileSync(potentialLicensePath, join(compiledPackagePath, './LICENSE'))
+    this._.files.push({
+      dir: compiledPackagePath,
+      base: 'LICENSE',
+      data: readFileSync(potentialLicensePath, 'utf8')
+    })
   }
 
   this._.files.push({

packages/next/taskfile.js

@@ -28,8 +28,16 @@ export async function ncc_unistore (task, opts) {
     .target('dist/compiled/unistore')
 }
 
+// eslint-disable-next-line camelcase
+export async function ncc_text_table (task, opts) {
+  await task
+    .source(opts.src || relative(__dirname, require.resolve('text-table')))
+    .ncc({ packageName: 'text-table' })
+    .target('dist/compiled/text-table')
+}
+
 export async function precompile (task) {
-  await task.parallel(['ncc_unistore'])
+  await task.parallel(['ncc_unistore', 'ncc_text_table'])
 }
 
 export async function compile (task) {

yarn.lock

@@ -1518,6 +1518,13 @@
   dependencies:
     chokidar "^1.7.0"
 
+"@types/amphtml-validator@1.0.0":
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/@types/amphtml-validator/-/amphtml-validator-1.0.0.tgz#9d4e0c879642938bbe5f363d49cafc8ae9f57c81"
+  integrity sha512-CJOi00fReT1JehItkgTZDI47v9WJxUH/OLX0XzkDgyEed7dGdeUQfXk5CTRM7N9FkHdv3klSjsZxo5sH1oTIGg==
+  dependencies:
+    "@types/node" "*"
+
 "@types/anymatch@*":
   version "1.3.1"
   resolved "https://registry.yarnpkg.com/@types/anymatch/-/anymatch-1.3.1.tgz#336badc1beecb9dacc38bea2cf32adf627a8421a"
@@ -1668,11 +1675,21 @@
   resolved "https://registry.yarnpkg.com/@types/source-list-map/-/source-list-map-0.1.2.tgz#0078836063ffaf17412349bba364087e0ac02ec9"
   integrity sha512-K5K+yml8LTo9bWJI/rECfIPrGgxdpeNbj+d53lwN4QjW1MCwlkhUms+gtdzigTeUyBr09+u8BwOIY3MXvHdcsA==
 
+"@types/strip-ansi@3.0.0":
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/@types/strip-ansi/-/strip-ansi-3.0.0.tgz#9b63d453a6b54aa849182207711a08be8eea48ae"
+  integrity sha1-m2PUU6a1SqhJGCIHcRoIvo7qSK4=
+
 "@types/tapable@*":
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/@types/tapable/-/tapable-1.0.4.tgz#b4ffc7dc97b498c969b360a41eee247f82616370"
   integrity sha512-78AdXtlhpCHT0K3EytMpn4JNxaf5tbqbLcbIRoQIHzpTIyjpxLQKRoxU55ujBXAtg3Nl2h/XWvfDa9dsMOd0pQ==
 
+"@types/text-table@0.2.0":
+  version "0.2.0"
+  resolved "https://registry.yarnpkg.com/@types/text-table/-/text-table-0.2.0.tgz#cec067b522598bbb37186375b834ac30411c9af6"
+  integrity sha512-om4hNWnI01IKUFCjGQG33JqFcnmt0W5C3WX0G1FVBaucr7oRnL29aAz2hnxpbZnE2t9f8/BR5VOtgcOtsonpLA==
+
 "@types/uglify-js@*":
   version "3.0.4"
   resolved "https://registry.yarnpkg.com/@types/uglify-js/-/uglify-js-3.0.4.tgz#96beae23df6f561862a830b4288a49e86baac082"
@@ -11914,7 +11931,7 @@ text-extensions@^1.0.0:
   resolved "https://registry.yarnpkg.com/text-extensions/-/text-extensions-1.9.0.tgz#1853e45fee39c945ce6f6c36b2d659b5aabc2a26"
   integrity sha512-wiBrwC1EhBelW12Zy26JeOUkQ5mRu+5o8rpsJk5+2t+Y5vE7e842qtZDQ2g1NpX/29HdyFeJ4nSIhI47ENSxlQ==
 
-text-table@^0.2.0:
+text-table@0.2.0, text-table@^0.2.0:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4"
   integrity sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=