Clean up the CI scripts and make it so that they can be run from anywhere (#20538)

This cleans up the ci scripts so that they can be run from an arbitrary directory, and so that they don't have any bash lint issues, and are more explicit about which dart/pub/dartanalyzer executable they run.

I also fixed the format script to take a "--fix" argument that will fix all of the formatting issues found, including trailing whitespace and gn files.

I added a warning to the license script about untracked/ignored files in the fluttter repo because those so often trip up the license script.

I added missing license information to the ci scripts too.

There's now a bit of boilerplate at the beginning of each script (the follow_links function) in order to reliably find the actual location of the script: I'd put it into a common file, except that that code would be needed to reliably find the common location too, so I needed to duplicate it. It's the same boilerplate as what is used in the flutter/flutter repo for the flutter and dart scripts.

I deleted the ci/test.sh script, since it seems to be obsolete (the test it tries to run doesn't exist anywhere).

ci/analyze.sh

 #!/bin/bash
-echo "Analyzing dart:ui library..."
+#
+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+set -e
+
+# Needed because if it is set, cd may print the path it changed to.
+unset CDPATH
 
-echo "Using analyzer from `which dartanalyzer`"
+# On Mac OS, readlink -f doesn't work, so follow_links traverses the path one
+# link at a time, and then cds into the link destination and find out where it
+# ends up.
+#
+# The function is enclosed in a subshell to avoid changing the working directory
+# of the caller.
+function follow_links() (
+  cd -P "$(dirname -- "$1")"
+  file="$PWD/$(basename -- "$1")"
+  while [[ -L "$file" ]]; do
+    cd -P "$(dirname -- "$file")"
+    file="$(readlink -- "$file")"
+    cd -P "$(dirname -- "$file")"
+    file="$PWD/$(basename -- "$file")"
+  done
+  echo "$file"
+)
 
-dartanalyzer --version
+SCRIPT_DIR=$(follow_links "$(dirname -- "${BASH_SOURCE[0]}")")
+SRC_DIR="$(cd "$SCRIPT_DIR/../.."; pwd -P)"
+FLUTTER_DIR="$SRC_DIR/flutter"
+DART_BIN="$SRC_DIR/third_party/dart/tools/sdks/dart-sdk/bin"
+PUB="$DART_BIN/pub"
+DART_ANALYZER="$DART_BIN/dartanalyzer"
 
-RESULTS=`dartanalyzer                                                          \
-  --options flutter/analysis_options.yaml                                      \
-  --enable-experiment=non-nullable                                             \
-  "$1out/host_debug_unopt/gen/sky/bindings/dart_ui/ui.dart"                    \
-  2>&1                                                                         \
-  | grep -Ev "No issues found!"                                                \
-  | grep -Ev "Analyzing.+out/host_debug_unopt/gen/sky/bindings/dart_ui/ui\.dart"`
+echo "Using analyzer from $DART_ANALYZER"
 
-echo "$RESULTS"
-if [ -n "$RESULTS" ]; then
-  echo "Failed."
-  exit 1;
-fi
+"$DART_ANALYZER" --version
+
+function analyze() (
+  local last_arg="${!#}"
+  local results
+  # Grep sets its return status to non-zero if it doesn't find what it's
+  # looking for.
+  set +e
+  results="$("$DART_ANALYZER" "$@" 2>&1 |
+    grep -Ev "No issues found!" |
+    grep -Ev "Analyzing.+$last_arg")"
+  set -e
+  echo "$results"
+  if [ -n "$results" ]; then
+    echo "Failed analysis of $last_arg"
+    return 1
+  else
+    echo "Success: no issues found in $last_arg"
+  fi
+  return 0
+)
+
+echo "Analyzing dart:ui library..."
+analyze \
+  --options "$FLUTTER_DIR/analysis_options.yaml" \
+  --enable-experiment=non-nullable \
+  "$SRC_DIR/out/host_debug_unopt/gen/sky/bindings/dart_ui/ui.dart"
 
 echo "Analyzing flutter_frontend_server..."
-RESULTS=`dartanalyzer                                                          \
-  --packages=flutter/flutter_frontend_server/.dart_tool/package_config.json    \
-  --options flutter/analysis_options.yaml                                      \
-  flutter/flutter_frontend_server                                              \
-  2>&1                                                                         \
-  | grep -Ev "No issues found!"                                                \
-  | grep -Ev "Analyzing.+frontend_server"`
-echo "$RESULTS"
-if [ -n "$RESULTS" ]; then
-  echo "Failed."
-  exit 1;
-fi
+analyze \
+  --packages="$FLUTTER_DIR/flutter_frontend_server/.dart_tool/package_config.json" \
+  --options "$FLUTTER_DIR/analysis_options.yaml" \
+  "$FLUTTER_DIR/flutter_frontend_server"
 
 echo "Analyzing tools/licenses..."
-(cd flutter/tools/licenses && pub get)
-RESULTS=`dartanalyzer                                                          \
-  --packages=flutter/tools/licenses/.dart_tool/package_config.json             \
-  --options flutter/tools/licenses/analysis_options.yaml                       \
-  flutter/tools/licenses                                                       \
-  2>&1                                                                         \
-  | grep -Ev "No issues found!"                                                \
-  | grep -Ev "Analyzing.+tools/licenses"`
-echo "$RESULTS"
-if [ -n "$RESULTS" ]; then
-  echo "Failed."
-  exit 1;
-fi
+(cd "$FLUTTER_DIR/tools/licenses" && "$PUB" get)
+analyze \
+  --packages="$FLUTTER_DIR/tools/licenses/.dart_tool/package_config.json" \
+  --options "$FLUTTER_DIR/tools/licenses/analysis_options.yaml" \
+  "$FLUTTER_DIR/tools/licenses"
 
 echo "Analyzing testing/dart..."
-flutter/tools/gn --unoptimized
-ninja -C out/host_debug_unopt sky_engine sky_services
-(cd flutter/testing/dart && pub get)
-RESULTS=`dartanalyzer                                                          \
-  --packages=flutter/testing/dart/.dart_tool/package_config.json               \
-  --options flutter/analysis_options.yaml                                      \
-  flutter/testing/dart                                                         \
-  2>&1                                                                         \
-  | grep -Ev "No issues found!"                                                \
-  | grep -Ev "Analyzing.+testing/dart"`
-echo "$RESULTS"
-if [ -n "$RESULTS" ]; then
-  echo "Failed."
-  exit 1;
-fi
+"$FLUTTER_DIR/tools/gn" --unoptimized
+ninja -C "$SRC_DIR/out/host_debug_unopt" sky_engine sky_services
+(cd "$FLUTTER_DIR/testing/dart" && "$PUB" get)
+analyze \
+  --packages="$FLUTTER_DIR/testing/dart/.dart_tool/package_config.json" \
+  --options "$FLUTTER_DIR/analysis_options.yaml" \
+  "$FLUTTER_DIR/testing/dart"
 
 echo "Analyzing testing/scenario_app..."
-(cd flutter/testing/scenario_app && pub get)
-RESULTS=`dartanalyzer                                                          \
-  --packages=flutter/testing/scenario_app/.dart_tool/package_config.json       \
-  --options flutter/analysis_options.yaml                                      \
-  flutter/testing/scenario_app                                                 \
-  2>&1                                                                         \
-  | grep -Ev "No issues found!"                                                \
-  | grep -Ev "Analyzing.+testing/scenario_app"`
-echo "$RESULTS"
-if [ -n "$RESULTS" ]; then
-  echo "Failed."
-  exit 1;
-fi
\ No newline at end of file
+(cd "$FLUTTER_DIR/testing/scenario_app" && "$PUB" get)
+analyze \
+  --packages="$FLUTTER_DIR/testing/scenario_app/.dart_tool/package_config.json" \
+  --options "$FLUTTER_DIR/analysis_options.yaml" \
+  "$FLUTTER_DIR/testing/scenario_app"

ci/bin/lint.dart

-/// Runs clang-tidy on files with changes.
-///
-/// usage:
-/// dart lint.dart <path to compile_commands.json> <path to git repository> [clang-tidy checks]
-///
-/// User environment variable FLUTTER_LINT_ALL to run on all files.
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Runs clang-tidy on files with changes.
+//
+// usage:
+// dart lint.dart <path to compile_commands.json> <path to git repository> [clang-tidy checks]
+//
+// User environment variable FLUTTER_LINT_ALL to run on all files.
 
 import 'dart:async' show Completer;
 import 'dart:convert' show jsonDecode, utf8, LineSplitter;

ci/build.sh

 #!/bin/bash
-set -ex
+#
+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
 
-PATH="$HOME/depot_tools:$PATH"
-cd ..
+set -e
 
-PATH=$(pwd)/third_party/dart/tools/sdks/dart-sdk/bin:$PATH
+# Needed because if it is set, cd may print the path it changed to.
+unset CDPATH
+
+# On Mac OS, readlink -f doesn't work, so follow_links traverses the path one
+# link at a time, and then cds into the link destination and find out where it
+# ends up.
+#
+# The function is enclosed in a subshell to avoid changing the working directory
+# of the caller.
+function follow_links() (
+  cd -P "$(dirname -- "$1")"
+  file="$PWD/$(basename -- "$1")"
+  while [[ -h "$file" ]]; do
+    cd -P "$(dirname -- "$file")"
+    file="$(readlink -- "$file")"
+    cd -P "$(dirname -- "$file")"
+    file="$PWD/$(basename -- "$file")"
+  done
+  echo "$file"
+)
+
+SCRIPT_DIR=$(follow_links "$(dirname -- "${BASH_SOURCE[0]}")")
+SRC_DIR="$(cd "$SCRIPT_DIR/../.."; pwd -P)"
+FLUTTER_DIR="$SRC_DIR/flutter"
+
+set -x
+
+PATH="$SRC_DIR/third_party/dart/tools/sdks/dart-sdk/bin:$HOME/depot_tools:$PATH"
+
+cd "$SRC_DIR"
 
 # Build the dart UI files
-flutter/tools/gn --unoptimized
-ninja -C out/host_debug_unopt generate_dart_ui
+"$FLUTTER_DIR/tools/gn" --unoptimized
+ninja -C "$SRC_DIR/out/host_debug_unopt" generate_dart_ui
 
 # Analyze the dart UI
-flutter/ci/analyze.sh
-flutter/ci/licenses.sh
+"$FLUTTER_DIR/ci/analyze.sh"
+"$FLUTTER_DIR/ci/licenses.sh"
 
 # Check that dart libraries conform
-cd flutter/web_sdk
-pub get
-cd ..
-dart web_sdk/test/api_conform_test.dart
+(cd "$FLUTTER_DIR/web_sdk"; pub get)
+(cd "$FLUTTER_DIR"; dart "web_sdk/test/api_conform_test.dart")
\ No newline at end of file

ci/check_gn_format.py

@@ -8,8 +8,6 @@ import sys
 import subprocess
 import os
 import argparse
-import errno
-import shutil
 
 def GetGNFiles(directory):
   directory = os.path.abspath(directory)

ci/check_roll.sh

 #!/bin/bash
+#
+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+set -e
+
+# Needed because if it is set, cd may print the path it changed to.
+unset CDPATH
+
+# On Mac OS, readlink -f doesn't work, so follow_links traverses the path one
+# link at a time, and then cds into the link destination and find out where it
+# ends up.
+#
+# The function is enclosed in a subshell to avoid changing the working directory
+# of the caller.
+function follow_links() (
+  cd -P "$(dirname -- "$1")"
+  file="$PWD/$(basename -- "$1")"
+  while [[ -h "$file" ]]; do
+    cd -P "$(dirname -- "$file")"
+    file="$(readlink -- "$file")"
+    cd -P "$(dirname -- "$file")"
+    file="$PWD/$(basename -- "$file")"
+  done
+  echo "$file"
+)
+
+SCRIPT_DIR=$(follow_links "$(dirname -- "${BASH_SOURCE[0]}")")
+FLUTTER_DIR="$(cd "$SCRIPT_DIR/.."; pwd -P)"
+
+cd "$FLUTTER_DIR"
 
 if git remote get-url upstream >/dev/null 2>&1; then
   UPSTREAM=upstream/master
@@ -7,7 +39,7 @@ else
 fi;
 
 FLUTTER_VERSION="$(curl -s https://raw.githubusercontent.com/flutter/flutter/master/bin/internal/engine.version)"
-BEHIND="$(git rev-list $FLUTTER_VERSION..$UPSTREAM --oneline | wc -l)"
+BEHIND="$(git rev-list "$FLUTTER_VERSION".."$UPSTREAM" --oneline | wc -l)"
 MAX_BEHIND=16 # no more than 4 bisections to identify the issue
 
 if [[ $BEHIND -le $MAX_BEHIND ]]; then
@@ -18,4 +50,3 @@ else
   echo "       please roll engine into flutter first before merging more commits into engine."
   exit 1
 fi
-

ci/firebase_testlab.sh

 #!/bin/bash
+#
+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
 
 set -e
 
-if [[ ! -f $1 ]]; then
-  echo "File $1 not found."
-  exit -1
+APP="$1"
+if [[ -z "$APP" ]]; then
+  echo "Application must be specified as the first argument to the script."
+  exit 255
 fi
 
-GIT_REVISION=${2:-$(git rev-parse HEAD)}
-BUILD_ID=${3:-$CIRRUS_BUILD_ID}
+if [[ ! -f "$APP" ]]; then
+  echo "File '$APP' not found."
+  exit 255
+fi
+
+GIT_REVISION="${2:-$(git rev-parse HEAD)}"
+BUILD_ID="${3:-$CIRRUS_BUILD_ID}"
 
-if [[ ! -z $GCLOUD_FIREBASE_TESTLAB_KEY ]]; then
+if [[ -n $GCLOUD_FIREBASE_TESTLAB_KEY ]]; then
   # New contributors will not have permissions to run this test - they won't be
   # able to access the service account information. We should just mark the test
   # as passed - it will run fine on post submit, where it will still catch
@@ -21,8 +31,8 @@ if [[ ! -z $GCLOUD_FIREBASE_TESTLAB_KEY ]]; then
     exit 0
   fi
 
-  echo $GCLOUD_FIREBASE_TESTLAB_KEY > ${HOME}/gcloud-service-key.json
-  gcloud auth activate-service-account --key-file=${HOME}/gcloud-service-key.json
+  echo "$GCLOUD_FIREBASE_TESTLAB_KEY" > "${HOME}/gcloud-service-key.json"
+  gcloud auth activate-service-account --key-file="${HOME}/gcloud-service-key.json"
 fi
 
 # Run the test.
@@ -32,8 +42,8 @@ fi
 # See https://firebase.google.com/docs/test-lab/android/game-loop
 gcloud --project flutter-infra firebase test android run \
   --type game-loop \
-  --app $1 \
+  --app "$APP" \
   --timeout 2m \
   --results-bucket=gs://flutter_firebase_testlab \
-  --results-dir=engine_scenario_test/$GIT_REVISION/$BUILD_ID \
+  --results-dir="engine_scenario_test/$GIT_REVISION/$BUILD_ID" \
   --no-auto-google-login

ci/format.sh

 #!/bin/bash
 #
-# Code formatting presubmit
+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Code formatting presubmit script.
 #
 # This presubmit script ensures that code under the src/flutter directory is
-# formatted according to the Flutter engine style requirements. On failure, a
-# diff is emitted that can be applied from within the src/flutter directory
-# via:
+# formatted according to the Flutter engine style requirements.
 #
-# patch -p0 < diff.patch
+# If failures are found, they can be fixed by re-running the script with the
+# --fix option.
 
 set -e
-echo "Checking formatting..."
 
-case "$(uname -s)" in
+# Needed because if it is set, cd may print the path it changed to.
+unset CDPATH
+
+# On Mac OS, readlink -f doesn't work, so follow_links traverses the path one
+# link at a time, and then cds into the link destination and find out where it
+# ends up.
+#
+# The function is enclosed in a subshell to avoid changing the working directory
+# of the caller.
+function follow_links() (
+  cd -P "$(dirname -- "$1")"
+  file="$PWD/$(basename -- "$1")"
+  while [[ -h "$file" ]]; do
+    cd -P "$(dirname -- "$file")"
+    file="$(readlink -- "$file")"
+    cd -P "$(dirname -- "$file")"
+    file="$PWD/$(basename -- "$file")"
+  done
+  echo "$file"
+)
+
+SCRIPT_DIR=$(follow_links "$(dirname -- "${BASH_SOURCE[0]}")")
+SRC_DIR="$(
+  cd "$SCRIPT_DIR/../.."
+  pwd -P
+)"
+FLUTTER_DIR="$SRC_DIR/flutter"
+
+function message() {
+  echo "$*" 1>&2
+}
+
+function error() {
+  echo "ERROR: $*" 1>&2
+}
+
+function warning() {
+  echo "WARNING: $*" 1>&2
+}
+
+function get_base_sha() (
+  local upstream
+  if git remote get-url upstream >/dev/null 2>&1; then
+    upstream=upstream
+  else
+    upstream=origin
+  fi
+
+  cd "$FLUTTER_DIR"
+  git fetch "$upstream" master >/dev/null 2>&1
+  git merge-base --fork-point FETCH_HEAD HEAD || git merge-base FETCH_HEAD HEAD
+)
+
+function check_clang_format() (
+  cd "$FLUTTER_DIR"
+  message "Checking C++/ObjC formatting..."
+
+  case "$(uname -s)" in
   Darwin)
     OS="mac-x64"
     ;;
@@ -20,81 +79,177 @@ case "$(uname -s)" in
     OS="linux-x64"
     ;;
   *)
-    echo "Unknown operating system."
-    exit -1
+    error "Unknown operating system."
+    return 255
     ;;
-esac
+  esac
 
-# Tools
-CLANG_FORMAT="../buildtools/$OS/clang/bin/clang-format"
-$CLANG_FORMAT --version
+  # Tools
+  local clang_format="$SRC_DIR/buildtools/$OS/clang/bin/clang-format"
+  "$clang_format" --version 1>&2
 
-# Compute the diffs.
-CLANG_FILETYPES="*.c *.cc *.cpp *.h *.m *.mm"
-DIFF_OPTS="-U0 --no-color --name-only"
+  local current_diff
+  local clang_files_to_check
 
-if git remote get-url upstream >/dev/null 2>&1; then
-  UPSTREAM=upstream
-else
-  UPSTREAM=origin
-fi;
-
-
-BASE_SHA="$(git fetch $UPSTREAM master > /dev/null 2>&1 && \
-           (git merge-base --fork-point FETCH_HEAD HEAD || git merge-base FETCH_HEAD HEAD))"
-# Disable glob matching otherwise a file in the current directory that matches
-# $CLANG_FILETYPES will cause git to query for that exact file instead of doing
-# a match.
-set -f
-CLANG_FILES_TO_CHECK="$(git ls-files $CLANG_FILETYPES)"
-set +f
-FAILED_CHECKS=0
-for f in $CLANG_FILES_TO_CHECK; do
-  set +e
-  CUR_DIFF="$(diff -u "$f" <("$CLANG_FORMAT" --style=file "$f"))"
-  set -e
-  if [[ ! -z "$CUR_DIFF" ]]; then
-    echo "$CUR_DIFF"
-    FAILED_CHECKS=$(($FAILED_CHECKS+1))
-  fi
-done
-
-GOOGLE_JAVA_FORMAT="../third_party/android_tools/google-java-format/google-java-format-1.7-all-deps.jar"
-if [[ -f "$GOOGLE_JAVA_FORMAT" && -f "$(which java)" ]]; then
-  java -jar "$GOOGLE_JAVA_FORMAT" --version 2>&1
-  JAVA_FILETYPES="*.java"
-  JAVA_FILES_TO_CHECK="$(git diff $DIFF_OPTS $BASE_SHA -- $JAVA_FILETYPES)"
-  for f in $JAVA_FILES_TO_CHECK; do
+  # Compute the diffs.
+  local clang_filetypes=("*.c" "*.cc" "*.cpp" "*.h" "*.m" "*.mm")
+  clang_files_to_check="$(git ls-files "${clang_filetypes[@]}")"
+  local failed_clang_checks=0
+  for file in $clang_files_to_check; do
     set +e
-    CUR_DIFF="$(diff -u "$f" <(java -jar "$GOOGLE_JAVA_FORMAT" "$f"))"
+    current_diff="$(diff -u "$file" <("$clang_format" --style=file "$file"))"
     set -e
-    if [[ ! -z "$CUR_DIFF" ]]; then
-      echo "$CUR_DIFF"
-      FAILED_CHECKS=$(($FAILED_CHECKS+1))
+    if [[ -n "$current_diff" ]]; then
+      echo "$current_diff"
+      failed_clang_checks=$((failed_clang_checks + 1))
     fi
   done
-else
-  echo "WARNING: Cannot find google-java-format, skipping Java file formatting!"
-fi
 
-if [[ $FAILED_CHECKS -ne 0 ]]; then
-  echo ""
-  echo "ERROR: Some files are formatted incorrectly. To fix, run \`./ci/format.sh | patch -p0\` from the flutter/engine/src/flutter directory."
-  exit 1
-fi
+  if [[ $failed_clang_checks -ne 0 ]]; then
+    error "$failed_clang_checks C++/ObjC files are formatted incorrectly."
+  fi
+  return $failed_clang_checks
+)
 
-FILETYPES="*.dart"
+function check_java_format() (
+  cd "$FLUTTER_DIR"
+  local diff_opts=("-U0" "--no-color" "--name-only")
+  message "Checking Java formatting..."
+  local google_java_format="$SRC_DIR/third_party/android_tools/google-java-format/google-java-format-1.7-all-deps.jar"
+  local failed_java_checks=0
+  if [[ -f "$google_java_format" && -n "$(command -v java)" ]]; then
+    java -jar "$google_java_format" --version 1>&2
+    local java_filetypes=("*.java")
+    local java_files_to_check
+    java_files_to_check="$(git diff "${diff_opts[@]}" "$BASE_SHA" -- "${java_filetypes[@]}")"
+    for file in $java_files_to_check; do
+      set +e
+      current_diff="$(diff -u "$file" <(java -jar "$google_java_format" "$file"))"
+      set -e
+      if [[ -n "$current_diff" ]]; then
+        echo "$current_diff"
+        failed_java_checks=$((failed_java_checks + 1))
+      fi
+    done
+    if [[ $failed_java_checks -ne 0 ]]; then
+      error "$failed_java_checks Java files are formatted incorrectly."
+    fi
+  else
+    warning "Cannot find google-java-format, skipping Java file formatting!"
+  fi
+  return $failed_java_checks
+)
 
-set +e
-TRAILING_SPACES=$(git diff $DIFF_OPTS $BASE_SHA..HEAD -- $FILETYPES | xargs grep --line-number --with-filename '[[:blank:]]\+$')
-set -e
+# Strips off the "advice" at the end, since this script has different advice.
+function do_gn_check() {
+  local output
+  output="$("$SCRIPT_DIR/check_gn_format.py" --dry-run --root-directory . --gn-binary "third_party/gn/gn")"
+  local result=$?
+  echo "$output" | grep "ERROR"
+  return $result
+}
 
-if [[ ! -z "$TRAILING_SPACES" ]]; then
-  echo "$TRAILING_SPACES"
-  echo ""
-  echo "ERROR: Some files have trailing spaces. To fix, try something like \`find . -name "*.dart" -exec sed -i -e 's/\s\+$//' {} \;\`."
-  exit 1
-fi
+function check_gn_format() (
+  cd "$FLUTTER_DIR"
+  message "Checking GN formatting..."
+  if ! do_gn_check 1>&2; then
+    error "The gn file format check failed."
+    return 1
+  fi
+)
+
+function check_whitespace() (
+  local diff_opts=("-U0" "--no-color" "--name-only")
+  local filetypes="*.dart"
+  local trailing_spaces
+
+  message "Checking for trailing whitespace in $filetypes files..." 1>&2
+  set +e
+  trailing_spaces=$(git diff "${diff_opts[@]}" "$BASE_SHA" -- "$filetypes" | xargs grep --line-number --with-filename '[[:blank:]]\+$')
+  set -e
+
+  if [[ -n "$trailing_spaces" ]]; then
+    message "$trailing_spaces"
+    error "Whitespace check failed. The above files have trailing spaces."
+    return 1
+  fi
+)
+
+function fix_clang_format() {
+  local tmpfile
+  tmpfile=$(mktemp "fix_clang_format.XXXXXX")
+  if check_clang_format >"$tmpfile"; then
+    message "No C++/ObjC formatting issues found."
+  else
+    message "Fixing C++/ObjC formatting issues."
+    (
+      cd "$SRC_DIR/flutter"
+      patch -p0 <"$tmpfile"
+    )
+  fi
+  command rm -f "$tmpfile"
+}
+
+function fix_java_format() {
+  local tmpfile
+  tmpfile=$(mktemp "fix_java_format.XXXXXX")
+  if check_java_format >"$tmpfile"; then
+    message "No Java formatting issues found."
+  else
+    message "Fixing Java formatting issues."
+    (
+      cd "$SRC_DIR/flutter"
+      patch -p0 <"$tmpfile"
+    )
+  fi
+  command rm -f "$tmpfile"
+}
 
-# Check GN format consistency
-./ci/check_gn_format.py --dry-run --root-directory . --gn-binary "third_party/gn/gn"
+function fix_gn_format() (
+  cd "$FLUTTER_DIR"
+  message "Fixing GN formatting..."
+  # Check GN format consistency and fix it.
+  if ! "$SCRIPT_DIR/check_gn_format.py" --root-directory . --gn-binary "third_party/gn/gn"; then
+    error "The GN file format fix failed."
+    return 1
+  fi
+)
+
+function fix_whitespace() {
+  if ! check_whitespace; then
+    message "Fixing trailing whitespace problems."
+    find "$FLUTTER_DIR" -type f -name "*.dart" -print0 | xargs -0 sed -i -e 's/\s\+$//'
+  fi
+}
+
+BASE_SHA=$(get_base_sha)
+message "Checking formatting for files with differences from $BASE_SHA in $FLUTTER_DIR"
+
+if [[ $1 == "--fix" ]]; then
+  fix_clang_format
+  fix_java_format
+  fix_gn_format
+  fix_whitespace
+  message "Formatting fixed."
+else
+  failures=0
+  if ! check_clang_format; then
+    failures=$((failures + 1))
+  fi
+  if ! check_java_format; then
+    failures=$((failures + 1))
+  fi
+  if ! check_gn_format; then
+    failures=$((failures + 1))
+  fi
+  if ! check_whitespace; then
+    failures=$((failures + 1))
+  fi
+  if [[ $failures -eq 0 ]]; then
+    message "No formatting issues found."
+  else
+    error "Formatting check failed."
+    error "To fix, run \"$SCRIPT_DIR/format.sh --fix\""
+  fi
+  exit $failures
+fi

ci/licenses.sh

 #!/bin/bash
+#
+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
 set -e
 shopt -s nullglob
 
+# Needed because if it is set, cd may print the path it changed to.
+unset CDPATH
+
+# On Mac OS, readlink -f doesn't work, so follow_links traverses the path one
+# link at a time, and then cds into the link destination and find out where it
+# ends up.
+#
+# The function is enclosed in a subshell to avoid changing the working directory
+# of the caller.
+function follow_links() (
+  cd -P "$(dirname -- "$1")"
+  file="$PWD/$(basename -- "$1")"
+  while [[ -h "$file" ]]; do
+    cd -P "$(dirname -- "$file")"
+    file="$(readlink -- "$file")"
+    cd -P "$(dirname -- "$file")"
+    file="$PWD/$(basename -- "$file")"
+  done
+  echo "$file"
+)
+
+SCRIPT_DIR=$(follow_links "$(dirname -- "${BASH_SOURCE[0]}")")
+SRC_DIR="$(cd "$SCRIPT_DIR/../.."; pwd -P)"
+DART_BIN="$SRC_DIR/third_party/dart/tools/sdks/dart-sdk/bin"
+PATH="$DART_BIN:$PATH"
+
 echo "Verifying license script is still happy..."
-echo "Using pub from `which pub`, dart from `which dart`"
+echo "Using pub from $(command -v pub), dart from $(command -v dart)"
 
-exitStatus=0
+untracked_files="$(cd "$SRC_DIR/flutter"; git status --ignored --short | grep -E "^!" | awk "{print\$2}")"
+untracked_count="$(echo "$untracked_files" | wc -l)"
+if [[ $untracked_count -gt 0 ]]; then
+  echo ""
+  echo "WARNING: There are $untracked_count untracked/ignored files or directories in the flutter repository."
+  echo "False positives may occur."
+  echo "You can use 'git clean -dxf' in the flutter dir to clean out these files."
+  echo "BUT, be warned that this will recursively remove all these files and directories:"
+  echo "$untracked_files"
+  echo ""
+fi
 
 dart --version
 
-# These files trip up the script on Mac OS X.
-find . -name ".DS_Store" -exec rm {} \;
-
-(cd flutter/tools/licenses; pub get; dart --enable-asserts lib/main.dart --src ../../.. --out ../../../out/license_script_output --golden ../../ci/licenses_golden)
-
-for f in out/license_script_output/licenses_*; do
-    if ! cmp -s flutter/ci/licenses_golden/$(basename $f) $f
-    then
-        echo "============================= ERROR ============================="
-        echo "License script got different results than expected for $f."
-        echo "Please rerun the licenses script locally to verify that it is"
-        echo "correctly catching any new licenses for anything you may have"
-        echo "changed, and then update this file:"
-        echo "  flutter/sky/packages/sky_engine/LICENSE"
-        echo "For more information, see the script in:"
-        echo "  https://github.com/flutter/engine/tree/master/tools/licenses"
-        echo ""
-        diff -U 6 flutter/ci/licenses_golden/$(basename $f) $f
-        echo "================================================================="
-        echo ""
-        exitStatus=1
-    fi
-done
-
-echo "Verifying license tool signature..."
-if ! cmp -s flutter/ci/licenses_golden/tool_signature out/license_script_output/tool_signature
-then
-    echo "============================= ERROR ============================="
-    echo "The license tool signature has changed. This is expected when"
-    echo "there have been changes to the license tool itself. Licenses have"
-    echo "been re-computed for all components. If only the license script has"
-    echo "changed, no diffs are typically expected in the output of the"
-    echo "script. Verify the output, and if it looks correct, update the"
-    echo "license tool signature golden file:"
-    echo "  ci/licenses_golden/tool_signature"
-    echo "For more information, see the script in:"
-    echo "  https://github.com/flutter/engine/tree/master/tools/licenses"
-    echo ""
-    diff -U 6 flutter/ci/licenses_golden/tool_signature out/license_script_output/tool_signature
-    echo "================================================================="
-    echo ""
-    exitStatus=1
-fi
+# Collects the license information from the repo.
+# Runs in a subshell.
+function collect_licenses() (
+  cd "$SRC_DIR/flutter/tools/licenses"
+  pub get
+  dart --enable-asserts lib/main.dart         \
+    --src ../../..                            \
+    --out ../../../out/license_script_output  \
+    --golden ../../ci/licenses_golden
+)
 
-echo "Checking license count in licenses_flutter..."
-actualLicenseCount=`tail -n 1 flutter/ci/licenses_golden/licenses_flutter | tr -dc '0-9'`
-expectedLicenseCount=2 # When changing this number: Update the error message below as well describing all expected license types.
-
-if [ "$actualLicenseCount" -ne "$expectedLicenseCount" ]
-then
-    echo "=============================== ERROR ==============================="
-    echo "The total license count in flutter/ci/licenses_golden/licenses_flutter"
-    echo "changed from $expectedLicenseCount to $actualLicenseCount."
-    echo "It's very likely that this is an unintentional change. Please"
-    echo "double-check that all newly added files have a BSD-style license"
-    echo "header with the following copyright:"
-    echo "    Copyright 2013 The Flutter Authors. All rights reserved."
-    echo "Files in 'third_party/txt' may have an Apache license header instead."
-    echo "If you're absolutely sure that the change in license count is"
-    echo "intentional, update 'flutter/ci/licenses.sh' with the new count."
-    echo "================================================================="
-    echo ""
-    exitStatus=1
-fi
+# Verifies the licenses in the repo.
+# Runs in a subshell.
+function verify_licenses() (
+  local exitStatus=0
+  cd "$SRC_DIR"
 
-if [ "$exitStatus" -eq "0" ]
-then
-  echo "Licenses are as expected."
-fi
-exit $exitStatus
+  # These files trip up the script on Mac OS X.
+  find . -name ".DS_Store" -exec rm {} \;
+
+  collect_licenses
+
+  for f in out/license_script_output/licenses_*; do
+      if ! cmp -s "flutter/ci/licenses_golden/$(basename "$f")" "$f"; then
+          echo "============================= ERROR ============================="
+          echo "License script got different results than expected for $f."
+          echo "Please rerun the licenses script locally to verify that it is"
+          echo "correctly catching any new licenses for anything you may have"
+          echo "changed, and then update this file:"
+          echo "  flutter/sky/packages/sky_engine/LICENSE"
+          echo "For more information, see the script in:"
+          echo "  https://github.com/flutter/engine/tree/master/tools/licenses"
+          echo ""
+          diff -U 6 "flutter/ci/licenses_golden/$(basename "$f")" "$f"
+          echo "================================================================="
+          echo ""
+          exitStatus=1
+      fi
+  done
+
+  echo "Verifying license tool signature..."
+  if ! cmp -s "flutter/ci/licenses_golden/tool_signature" "out/license_script_output/tool_signature"; then
+      echo "============================= ERROR ============================="
+      echo "The license tool signature has changed. This is expected when"
+      echo "there have been changes to the license tool itself. Licenses have"
+      echo "been re-computed for all components. If only the license script has"
+      echo "changed, no diffs are typically expected in the output of the"
+      echo "script. Verify the output, and if it looks correct, update the"
+      echo "license tool signature golden file:"
+      echo "  ci/licenses_golden/tool_signature"
+      echo "For more information, see the script in:"
+      echo "  https://github.com/flutter/engine/tree/master/tools/licenses"
+      echo ""
+      diff -U 6 "flutter/ci/licenses_golden/tool_signature" "out/license_script_output/tool_signature"
+      echo "================================================================="
+      echo ""
+      exitStatus=1
+  fi
+
+  echo "Checking license count in licenses_flutter..."
+
+  local actualLicenseCount
+  actualLicenseCount="$(tail -n 1 flutter/ci/licenses_golden/licenses_flutter | tr -dc '0-9')"
+  local expectedLicenseCount=2 # When changing this number: Update the error message below as well describing all expected license types.
+
+  if [[ $actualLicenseCount -ne $expectedLicenseCount ]]; then
+      echo "=============================== ERROR ==============================="
+      echo "The total license count in flutter/ci/licenses_golden/licenses_flutter"
+      echo "changed from $expectedLicenseCount to $actualLicenseCount."
+      echo "It's very likely that this is an unintentional change. Please"
+      echo "double-check that all newly added files have a BSD-style license"
+      echo "header with the following copyright:"
+      echo "    Copyright 2013 The Flutter Authors. All rights reserved."
+      echo "Files in 'third_party/txt' may have an Apache license header instead."
+      echo "If you're absolutely sure that the change in license count is"
+      echo "intentional, update 'flutter/ci/licenses.sh' with the new count."
+      echo "================================================================="
+      echo ""
+      exitStatus=1
+  fi
+
+  if [[ $exitStatus -eq 0 ]]; then
+    echo "Licenses are as expected."
+  fi
+  return $exitStatus
+)
+
+verify_licenses
\ No newline at end of file

ci/lint.sh

 #!/bin/bash
+#
+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
 
 set -e
-SCRIPT_DIR=`realpath $(dirname "${BASH_SOURCE[0]}")`
-DART="${SCRIPT_DIR}/../../third_party/dart/tools/sdks/dart-sdk/bin/dart"
-PUB="${SCRIPT_DIR}/../../third_party/dart/tools/sdks/dart-sdk/bin/pub"
 
 # Needed because if it is set, cd may print the path it changed to.
 unset CDPATH
@@ -12,15 +13,6 @@ unset CDPATH
 # link at a time, and then cds into the link destination and find out where it
 # ends up.
 #
-# The returned filesystem path must be a format usable by Dart's URI parser,
-# since the Dart command line tool treats its argument as a file URI, not a
-# filename. For instance, multiple consecutive slashes should be reduced to a
-# single slash, since double-slashes indicate a URI "authority", and these are
-# supposed to be filenames. There is an edge case where this will return
-# multiple slashes: when the input resolves to the root directory. However, if
-# that were the case, we wouldn't be running this shell, so we don't do anything
-# about it.
-#
 # The function is enclosed in a subshell to avoid changing the working directory
 # of the caller.
 function follow_links() (
@@ -34,17 +26,20 @@ function follow_links() (
   done
   echo "$file"
 )
-PROG_NAME="$(follow_links "${BASH_SOURCE[0]}")"
-CI_DIR="$(cd "${PROG_NAME%/*}" ; pwd -P)"
-SRC_DIR="$(cd "$CI_DIR/../.."; pwd -P)"
+
+SCRIPT_DIR=$(follow_links "$(dirname -- "${BASH_SOURCE[0]}")")
+SRC_DIR="$(cd "$SCRIPT_DIR/../.."; pwd -P)"
+DART_BIN="${SRC_DIR}/third_party/dart/tools/sdks/dart-sdk/bin"
+DART="${DART_BIN}/dart"
+PUB="${DART_BIN}/pub"
 
 COMPILE_COMMANDS="$SRC_DIR/out/compile_commands.json"
 if [ ! -f "$COMPILE_COMMANDS" ]; then
-  (cd $SRC_DIR; ./flutter/tools/gn)
+  (cd "$SRC_DIR"; ./flutter/tools/gn)
 fi
 
-cd "$CI_DIR"
-$PUB get && $DART \
+cd "$SCRIPT_DIR"
+"$PUB" get && "$DART" \
   --disable-dart-dev \
   bin/lint.dart \
   --compile-commands="$COMPILE_COMMANDS" \

ci/pubspec.yaml

+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
 name: ci_scripts
 
 dependencies:
   args: ^1.6.0
   path: ^1.7.0
-  process_runner: ^2.0.3
+  process_runner: ^3.0.0
 
 environment:
   sdk: '>=2.8.0 <3.0.0'

ci/test.sh

-#!/bin/bash
-
-cd frontend_server
-dart test/server_test.dart