Convert format script to Dart (#20572)

This converts the ci/format.sh script to a Dart script that uses process_runner and isolates to multi-process the clang-format, diffs, and grepping needed to do the formatting changes.

It also will (by default) only check the formatting of changed files.

The user can optionally check all files (--all-files) or do only some types of checks with --check. --verbose prints the versions of the tools used for Clang format and Java format.

Specifying --fix will cause any formatting errors that would have been detected to be fixed.

build/generate_coverage.py

@@ -42,7 +42,7 @@ def RemoveIfExists(path):
 
 def main():
   parser = argparse.ArgumentParser();
-  
+
   parser.add_argument('-t', '--tests', nargs='+', dest='tests',
       required=True, help='The unit tests to run and gather coverage data on.')
   parser.add_argument('-o', '--output', dest='output',
@@ -64,19 +64,19 @@ def main():
   # Run all unit tests and collect raw profiles.
   for test in args.tests:
     absolute_test_path = os.path.abspath(test)
-    
+
     if not os.path.exists(absolute_test_path):
       print("Path %s does not exist." % absolute_test_path)
       return -1
 
     binaries.append(absolute_test_path)
-    
+
     raw_profile = absolute_test_path + ".rawprofile"
 
     RemoveIfExists(raw_profile)
 
     print "Running test %s to gather profile." % os.path.basename(absolute_test_path)
-    
+
     subprocess.check_call([absolute_test_path], env={
       "LLVM_PROFILE_FILE":  raw_profile
     })

ci/check_gn_format.py

-#!/usr/bin/env python
-#
-# Copyright 2013 The Flutter Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import sys
-import subprocess
-import os
-import argparse
-
-def GetGNFiles(directory):
-  directory = os.path.abspath(directory)
-  gn_files = []
-  assert os.path.exists(directory), "Directory must exist %s" % directory
-  for root, dirs, files in os.walk(directory):
-    for file in files:
-        if file.endswith(".gn") or file.endswith(".gni"):
-          gn_files.append(os.path.join(root, file))
-  return gn_files
-
-def main():
-  parser = argparse.ArgumentParser();
-
-  parser.add_argument('--gn-binary', dest='gn_binary', required=True, type=str)
-  parser.add_argument('--dry-run', dest='dry_run', default=False, action='store_true')
-  parser.add_argument('--root-directory', dest='root_directory', required=True, type=str)
-
-  args = parser.parse_args()
-
-  gn_binary = os.path.abspath(args.gn_binary)
-  assert os.path.exists(gn_binary), "GN Binary must exist %s" % gn_binary
-
-  gn_command = [ gn_binary, 'format']
-
-  if args.dry_run:
-    gn_command.append('--dry-run')
-
-  for gn_file in GetGNFiles(args.root_directory):
-    if subprocess.call(gn_command + [ gn_file ]) != 0:
-      print "ERROR: '%s' is incorrectly formatted." % os.path.relpath(gn_file, args.root_directory)
-      print "Format the same with 'gn format' using the 'gn' binary in third_party/gn/gn."
-      print "Or, run ./ci/check_gn_format.py without '--dry-run'"
-      return 1
-
-  return 0
-
-if __name__ == '__main__':
-  sys.exit(main())

ci/format.bat

+@ECHO off
+REM Copyright 2013 The Flutter Authors. All rights reserved.
+REM Use of this source code is governed by a BSD-style license that can be
+REM found in the LICENSE file.
+
+REM ---------------------------------- NOTE ----------------------------------
+REM
+REM Please keep the logic in this file consistent with the logic in the
+REM `format.sh` script in the same directory to ensure that it continues to
+REM work across all platforms!
+REM
+REM --------------------------------------------------------------------------
+
+SETLOCAL ENABLEDELAYEDEXPANSION
+
+FOR %%i IN ("%~dp0..\..") DO SET SRC_DIR=%%~fi
+
+REM Test if Git is available on the Host
+where /q git || ECHO Error: Unable to find git in your PATH. && EXIT /B 1
+
+SET repo_dir=%SRC_DIR%\flutter
+SET ci_dir=%repo_dir%\flutter\ci
+SET dart_sdk_path=%SRC_DIR%\third_party\dart\tools\sdks\dart-sdk
+SET dart=%dart_sdk_path%\bin\dart.exe
+SET pub=%dart_sdk_path%\bin\pub.bat
+
+cd "%ci_dir%"
+
+REM Do not use the CALL command in the next line to execute Dart. CALL causes
+REM Windows to re-read the line from disk after the CALL command has finished
+REM regardless of the ampersand chain.
+"%pub%" get & "%dart%" --disable-dart-dev bin\format.dart %* & exit /B !ERRORLEVEL!

ci/format.sh

@@ -4,14 +4,6 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-# Code formatting presubmit script.
-#
-# This presubmit script ensures that code under the src/flutter directory is
-# formatted according to the Flutter engine style requirements.
-#
-# If failures are found, they can be fixed by re-running the script with the
-# --fix option.
-
 set -e
 
 # Needed because if it is set, cd may print the path it changed to.
@@ -35,221 +27,15 @@ function follow_links() (
   echo "$file"
 )
 
-SCRIPT_DIR=$(follow_links "$(dirname -- "${BASH_SOURCE[0]}")")
-SRC_DIR="$(
-  cd "$SCRIPT_DIR/../.."
-  pwd -P
-)"
-FLUTTER_DIR="$SRC_DIR/flutter"
-
-function message() {
-  echo "$*" 1>&2
-}
-
-function error() {
-  echo "ERROR: $*" 1>&2
-}
-
-function warning() {
-  echo "WARNING: $*" 1>&2
-}
-
-function get_base_sha() (
-  local upstream
-  if git remote get-url upstream >/dev/null 2>&1; then
-    upstream=upstream
-  else
-    upstream=origin
-  fi
-
-  cd "$FLUTTER_DIR"
-  git fetch "$upstream" master >/dev/null 2>&1
-  git merge-base --fork-point FETCH_HEAD HEAD || git merge-base FETCH_HEAD HEAD
-)
-
-function check_clang_format() (
-  cd "$FLUTTER_DIR"
-  message "Checking C++/ObjC formatting..."
-
-  case "$(uname -s)" in
-  Darwin)
-    OS="mac-x64"
-    ;;
-  Linux)
-    OS="linux-x64"
-    ;;
-  *)
-    error "Unknown operating system."
-    return 255
-    ;;
-  esac
-
-  # Tools
-  local clang_format="$SRC_DIR/buildtools/$OS/clang/bin/clang-format"
-  "$clang_format" --version 1>&2
-
-  local current_diff
-  local clang_files_to_check
-
-  # Compute the diffs.
-  local clang_filetypes=("*.c" "*.cc" "*.cpp" "*.h" "*.m" "*.mm")
-  clang_files_to_check="$(git ls-files "${clang_filetypes[@]}")"
-  local failed_clang_checks=0
-  for file in $clang_files_to_check; do
-    set +e
-    current_diff="$(diff -u "$file" <("$clang_format" --style=file "$file"))"
-    set -e
-    if [[ -n "$current_diff" ]]; then
-      echo "$current_diff"
-      failed_clang_checks=$((failed_clang_checks + 1))
-    fi
-  done
-
-  if [[ $failed_clang_checks -ne 0 ]]; then
-    error "$failed_clang_checks C++/ObjC files are formatted incorrectly."
-  fi
-  return $failed_clang_checks
-)
-
-function check_java_format() (
-  cd "$FLUTTER_DIR"
-  local diff_opts=("-U0" "--no-color" "--name-only")
-  message "Checking Java formatting..."
-  local google_java_format="$SRC_DIR/third_party/android_tools/google-java-format/google-java-format-1.7-all-deps.jar"
-  local failed_java_checks=0
-  if [[ -f "$google_java_format" && -n "$(command -v java)" ]]; then
-    java -jar "$google_java_format" --version 1>&2
-    local java_filetypes=("*.java")
-    local java_files_to_check
-    java_files_to_check="$(git diff "${diff_opts[@]}" "$BASE_SHA" -- "${java_filetypes[@]}")"
-    for file in $java_files_to_check; do
-      set +e
-      current_diff="$(diff -u "$file" <(java -jar "$google_java_format" "$file"))"
-      set -e
-      if [[ -n "$current_diff" ]]; then
-        echo "$current_diff"
-        failed_java_checks=$((failed_java_checks + 1))
-      fi
-    done
-    if [[ $failed_java_checks -ne 0 ]]; then
-      error "$failed_java_checks Java files are formatted incorrectly."
-    fi
-  else
-    warning "Cannot find google-java-format, skipping Java file formatting!"
-  fi
-  return $failed_java_checks
-)
 
-# Strips off the "advice" at the end, since this script has different advice.
-function do_gn_check() {
-  local output
-  output="$("$SCRIPT_DIR/check_gn_format.py" --dry-run --root-directory . --gn-binary "third_party/gn/gn")"
-  local result=$?
-  echo "$output" | grep "ERROR"
-  return $result
-}
-
-function check_gn_format() (
-  cd "$FLUTTER_DIR"
-  message "Checking GN formatting..."
-  if ! do_gn_check 1>&2; then
-    error "The gn file format check failed."
-    return 1
-  fi
-)
-
-function check_whitespace() (
-  local diff_opts=("-U0" "--no-color" "--name-only")
-  local filetypes="*.dart"
-  local trailing_spaces
-
-  message "Checking for trailing whitespace in $filetypes files..." 1>&2
-  set +e
-  trailing_spaces=$(git diff "${diff_opts[@]}" "$BASE_SHA" -- "$filetypes" | xargs grep --line-number --with-filename '[[:blank:]]\+$')
-  set -e
-
-  if [[ -n "$trailing_spaces" ]]; then
-    message "$trailing_spaces"
-    error "Whitespace check failed. The above files have trailing spaces."
-    return 1
-  fi
-)
-
-function fix_clang_format() {
-  local tmpfile
-  tmpfile=$(mktemp "fix_clang_format.XXXXXX")
-  if check_clang_format >"$tmpfile"; then
-    message "No C++/ObjC formatting issues found."
-  else
-    message "Fixing C++/ObjC formatting issues."
-    (
-      cd "$SRC_DIR/flutter"
-      patch -p0 <"$tmpfile"
-    )
-  fi
-  command rm -f "$tmpfile"
-}
-
-function fix_java_format() {
-  local tmpfile
-  tmpfile=$(mktemp "fix_java_format.XXXXXX")
-  if check_java_format >"$tmpfile"; then
-    message "No Java formatting issues found."
-  else
-    message "Fixing Java formatting issues."
-    (
-      cd "$SRC_DIR/flutter"
-      patch -p0 <"$tmpfile"
-    )
-  fi
-  command rm -f "$tmpfile"
-}
-
-function fix_gn_format() (
-  cd "$FLUTTER_DIR"
-  message "Fixing GN formatting..."
-  # Check GN format consistency and fix it.
-  if ! "$SCRIPT_DIR/check_gn_format.py" --root-directory . --gn-binary "third_party/gn/gn"; then
-    error "The GN file format fix failed."
-    return 1
-  fi
-)
-
-function fix_whitespace() {
-  if ! check_whitespace; then
-    message "Fixing trailing whitespace problems."
-    find "$FLUTTER_DIR" -type f -name "*.dart" -print0 | xargs -0 sed -i -e 's/\s\+$//'
-  fi
-}
-
-BASE_SHA=$(get_base_sha)
-message "Checking formatting for files with differences from $BASE_SHA in $FLUTTER_DIR"
-
-if [[ $1 == "--fix" ]]; then
-  fix_clang_format
-  fix_java_format
-  fix_gn_format
-  fix_whitespace
-  message "Formatting fixed."
-else
-  failures=0
-  if ! check_clang_format; then
-    failures=$((failures + 1))
-  fi
-  if ! check_java_format; then
-    failures=$((failures + 1))
-  fi
-  if ! check_gn_format; then
-    failures=$((failures + 1))
-  fi
-  if ! check_whitespace; then
-    failures=$((failures + 1))
-  fi
-  if [[ $failures -eq 0 ]]; then
-    message "No formatting issues found."
-  else
-    error "Formatting check failed."
-    error "To fix, run \"$SCRIPT_DIR/format.sh --fix\""
-  fi
-  exit $failures
-fi
+SCRIPT_DIR=$(follow_links "$(dirname -- "${BASH_SOURCE[0]}")")
+SRC_DIR="$(cd "$SCRIPT_DIR/../.."; pwd -P)"
+DART_SDK_DIR="${SRC_DIR}/third_party/dart/tools/sdks/dart-sdk"
+DART="${DART_SDK_DIR}/bin/dart"
+PUB="${DART_SDK_DIR}/bin/pub"
+
+cd "$SCRIPT_DIR"
+"$PUB" get && "$DART" \
+  --disable-dart-dev \
+  bin/format.dart \
+  "$@"

ci/pubspec.yaml

@@ -7,6 +7,7 @@ name: ci_scripts
 dependencies:
   args: ^1.6.0
   path: ^1.7.0
+  isolate: ^2.0.3
   process_runner: ^3.0.0
 
 environment:

testing/scenario_app/android/build.gradle

@@ -4,12 +4,11 @@ buildscript {
     repositories {
         google()
         jcenter()
-        
     }
     dependencies {
         classpath 'com.android.tools.build:gradle:3.6.0'
         classpath 'com.facebook.testing.screenshot:plugin:0.12.0'
-        
+
         // NOTE: Do not place your application dependencies here; they belong
         // in the individual module build.gradle files
     }