E2e screenshot tests2 (#21383)

* carrying code

* more changes for carrying the code

* rebase changes onto ios-screenshot tests

* adding screenshot capability to text_editing e2e test

* address some comments

* change enable flag for isUnitTestsScreenshotsAvailable

* addressing the reviewer comments

* change the dependency for path

* add to licencense file

* changing goldens commit no. the new commit has the screenshot goldens

* update readme file

* firefox tests needs LUCI changes

* change to release mode since screenshots were taken in release mode

* change window size

* some argument changes

* small comment change

* test the chrome linux tests again

* use roboto font instead of default font

* addressing reviewer comments

* change commit for goldens

ci/licenses_golden/licenses_flutter

@@ -1514,6 +1514,10 @@ FILE: ../../../flutter/vulkan/vulkan_window.cc
 FILE: ../../../flutter/vulkan/vulkan_window.h
 FILE: ../../../flutter/web_sdk/libraries.json
 FILE: ../../../flutter/web_sdk/sdk_rewriter.dart
+FILE: ../../../flutter/web_sdk/web_test_utils/lib/environment.dart
+FILE: ../../../flutter/web_sdk/web_test_utils/lib/exceptions.dart
+FILE: ../../../flutter/web_sdk/web_test_utils/lib/goldens.dart
+FILE: ../../../flutter/web_sdk/web_test_utils/lib/image_compare.dart
 ----------------------------------------------------------------------------------------------------
 Copyright 2013 The Flutter Authors. All rights reserved.
 

e2etests/web/regular_integration_tests/README.md

@@ -37,3 +37,34 @@ flutter drive -v --target=test_driver/text_editing_integration.dart -d web-serve
 ```
 
 More details for "Running Flutter Driver tests with Web" can be found in [wiki](https://github.com/flutter/flutter/wiki/Running-Flutter-Driver-tests-with-Web).
+
+## Adding screenshot tests
+
+In order to test screenshot tests the tests on the driver side needs to call the `integration_test` package with an `onScreenshot` callback which can do a comparison between the `screenshotBytes` taken during the test and a golden file. We added a utility method that can do this comparison by using a golden in `flutter/goldens` repository.
+
+In order to use screenshot testing first, import `screenshot_support.dart` from the driver side test (example: `text_editing_integration_test.dart`). Default value for `diffRateFailure` is 0.5.
+
+```
+import 'package:regular_integration_tests/screenshot_support.dart' as test;
+
+Future<void> main() async {
+  final double kMaxDiffRateFailure = 0.1;
+  await test.runTestWithScreenshots(diffRateFailure = kMaxDiffRateFailure);
+}
+```
+
+In order to run the tests follow these steps:
+
+1. You can use two different approaches, using [felt](https://github.com/flutter/engine/blob/master/lib/web_ui/dev/README.md) tool will run all the tests, an update all the goldens. For running individual tests, we need to set UPDATE_GOLDENS environment variable.
+
+```
+felt test --integration-tests-only --update-screenshot-goldens
+```
+
+```
+UPDATE_GOLDENS=true flutter drive -v --target=test_driver/text_editing_integration.dart -d web-server --release --local-engine=host_debug_unopt
+```
+
+2. The golden will be under `engine/src/flutter/lib/web_ui/.dart_tool/goldens/engine/web/` directory, you should create a PR for that file and merge it to `flutter/goldens`.
+
+3. Get the commit SHA and replace the `revision` in this file: `engine/src/flutter/lib/web_ui/dev/goldens_lock.yaml`

e2etests/web/regular_integration_tests/lib/screenshot_support.dart

+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io' as io;
+import 'dart:math';
+
+import 'package:flutter_driver/flutter_driver.dart';
+import 'package:integration_test/integration_test_driver_extended.dart' as test;
+
+import 'package:web_test_utils/goldens.dart';
+import 'package:web_test_utils/image_compare.dart';
+import 'package:webdriver/src/async/window.dart';
+
+import 'package:image/image.dart';
+
+/// Tolerable pixel difference ratio between the goldens and the screenshots.
+///
+/// We are allowing a higher difference rate compared to the unit tests (where
+/// this rate is set to 0.28), since during the end to end tests there are
+/// more components on the screen which are not related to the functionality
+/// under test ex: a blinking cursor.
+const double kMaxDiffRateFailure = 0.5 / 100; // 0.5%
+
+/// SBrowser screen dimensions for the Flutter Driver test.
+const int _kScreenshotWidth = 1024;
+const int _kScreenshotHeight = 1024;
+
+/// Used for calling `integration_test` package.
+///
+/// Compared to other similar classes which only included the following call:
+/// ```
+/// Future<void> main() async => test.integrationDriver();
+/// ```
+///
+/// this method is able to take screenshot.
+///
+/// It provides an `onScreenshot` callback to the `integrationDriver` method.
+/// It also includes options for updating the golden files.
+Future<void> runTestWithScreenshots(
+    {double diffRateFailure = kMaxDiffRateFailure,
+    int browserWidth = _kScreenshotWidth,
+    int browserHeight = _kScreenshotHeight}) async {
+  final WebFlutterDriver driver =
+      await FlutterDriver.connect() as WebFlutterDriver;
+
+  // Learn the browser in use from the webDriver.
+  final String browser = driver.webDriver.capabilities['browserName'] as String;
+
+  final Window window = await driver.webDriver.window;
+  window.setSize(Rectangle<int>(0, 0, browserWidth, browserHeight));
+
+  bool updateGoldens = false;
+  // We are using an environment variable instead of an argument, since
+  // this code is not invoked from the shell but from the `flutter drive`
+  // tool itself. Therefore we do not have control on the command line
+  // arguments.
+  // Please read the README, further info on how to update the goldens.
+  final String updateGoldensFlag = io.Platform.environment['UPDATE_GOLDENS'];
+  // Validate if the environment variable is set correctly.
+  if (updateGoldensFlag != null &&
+      !(updateGoldensFlag.toLowerCase() == 'true' ||
+          updateGoldensFlag.toLowerCase() == 'false')) {
+    throw StateError(
+        'UPDATE_GOLDENS environment variable is not set correctly');
+  }
+  if (updateGoldensFlag != null && updateGoldensFlag.toLowerCase() == 'true') {
+    updateGoldens = true;
+  }
+
+  test.integrationDriver(
+    driver: driver,
+    onScreenshot: (String screenshotName, List<int> screenshotBytes) async {
+      if (browser == 'chrome') {
+        final Image screenshot = decodePng(screenshotBytes);
+        final String result = compareImage(
+          screenshot,
+          updateGoldens,
+          '$screenshotName-$browser.png',
+          PixelComparison.fuzzy,
+          diffRateFailure,
+          forIntegrationTests: true,
+          write: updateGoldens,
+        );
+        if (result == 'OK') {
+          return true;
+        } else {
+          io.stderr.writeln('ERROR: $result');
+          return false;
+        }
+      } else {
+        return true;
+      }
+    },
+  );
+}

e2etests/web/regular_integration_tests/lib/text_editing_main.dart

@@ -11,6 +11,7 @@ class MyApp extends StatelessWidget {
   Widget build(BuildContext context) {
     return MaterialApp(
       key: const Key('mainapp'),
+      theme: ThemeData(fontFamily: 'RobotoMono'),
       title: 'Integration Test App',
       home: MyHomePage(title: 'Integration Test App'),
     );
@@ -56,6 +57,7 @@ class _MyHomePageState extends State<MyHomePage> {
               enabled: true,
               controller: _emptyController,
               decoration: const InputDecoration(
+                contentPadding: EdgeInsets.all(10.0),
                 labelText: 'Empty Input Field:',
               ),
             ),
@@ -67,6 +69,7 @@ class _MyHomePageState extends State<MyHomePage> {
               enabled: true,
               controller: _controller,
               decoration: const InputDecoration(
+                contentPadding: EdgeInsets.all(10.0),
                 labelText: 'Text Input Field:',
               ),
             ),
@@ -78,6 +81,7 @@ class _MyHomePageState extends State<MyHomePage> {
               enabled: true,
               controller: _controller2,
               decoration: const InputDecoration(
+                contentPadding: EdgeInsets.all(10.0),
                 labelText: 'Text Input Field 2:',
               ),
               onFieldSubmitted: (String str) {
@@ -94,7 +98,7 @@ class _MyHomePageState extends State<MyHomePage> {
               child: SelectableText(
                 'Lorem ipsum dolor sit amet',
                 key: Key('selectable'),
-                style: TextStyle(fontFamily: 'Roboto', fontSize: 20.0),
+                style: TextStyle(fontFamily: 'RobotoMono', fontSize: 20.0),
               ),
             ),
           ],

e2etests/web/regular_integration_tests/pubspec.yaml

@@ -15,8 +15,14 @@ dev_dependencies:
     sdk: flutter
   integration_test: 0.9.0
   http: 0.12.0+2
-  test: any
+  web_test_utils:
+    path: ../../../web_sdk/web_test_utils
 
 flutter:
   assets:
     - assets/images/
+  fonts:
+    - family: RobotoMono
+      fonts:
+        - asset: fonts/RobotoMono-Bold.ttf
+        - asset: fonts/RobotoMono-Regular.ttf

e2etests/web/regular_integration_tests/test_driver/text_editing_integration.dart

@@ -13,7 +13,7 @@ import 'package:flutter/material.dart';
 import 'package:integration_test/integration_test.dart';
 
 void main() {
-  IntegrationTestWidgetsFlutterBinding.ensureInitialized();
+  final IntegrationTestWidgetsFlutterBinding binding = IntegrationTestWidgetsFlutterBinding.ensureInitialized() as IntegrationTestWidgetsFlutterBinding;
 
   testWidgets('Focused text field creates a native input element',
       (WidgetTester tester) async {
@@ -41,6 +41,8 @@ void main() {
     textFormField.controller.text = 'New Value';
     // DOM element's value also changes.
     expect(input.value, 'New Value');
+
+    await binding.takeScreenshot('focused_text_field');
   });
 
   testWidgets('Input field with no initial value works',

e2etests/web/regular_integration_tests/test_driver/text_editing_integration_test.dart

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import 'package:integration_test/integration_test_driver.dart' as test;
+import 'package:regular_integration_tests/screenshot_support.dart' as test;
 
-Future<void> main() async => test.integrationDriver();
+Future<void> main() async {
+  await test.runTestWithScreenshots();
+}

lib/web_ui/dev/goldens_lock.yaml

 repository: https://github.com/flutter/goldens.git
-revision: 1a4722227af42c3f51450266016b1a07ae459e73
+revision: da3fef0c0eb849dfbb14b09a088c5f7916677482

lib/web_ui/dev/integration_tests_manager.dart

@@ -24,7 +24,10 @@ class IntegrationTestsManager {
 
   final DriverManager _driverManager;
 
-  IntegrationTestsManager(this._browser, this._useSystemFlutter)
+  final bool _doUpdateScreenshotGoldens;
+
+  IntegrationTestsManager(
+      this._browser, this._useSystemFlutter, this._doUpdateScreenshotGoldens)
       : _driverManager = DriverManager.chooseDriver(_browser);
 
   Future<bool> runTests() async {
@@ -159,14 +162,19 @@ class IntegrationTestsManager {
 
   Future<bool> _runTestsInProfileMode(
       io.Directory directory, String testName) async {
-    final String executable =
+    String executable =
         _useSystemFlutter ? 'flutter' : environment.flutterCommand.path;
+    Map<String, String> enviroment = Map<String, String>();
+    if (_doUpdateScreenshotGoldens) {
+      enviroment['UPDATE_GOLDENS'] = 'true';
+    }
     final IntegrationArguments arguments =
         IntegrationArguments.fromBrowser(_browser);
     final int exitCode = await runProcess(
       executable,
       arguments.getTestArguments(testName, 'profile'),
       workingDirectory: directory.path,
+      environment: enviroment,
     );
 
     if (exitCode != 0) {
@@ -334,7 +342,7 @@ class ChromeIntegrationArguments extends IntegrationArguments {
       '--$mode',
       '--browser-name=chrome',
       if (isLuci) '--chrome-binary=${preinstalledChromeExecutable()}',
-      if (isLuci) '--headless',
+      '--headless',
       '--local-engine=host_debug_unopt',
     ];
   }

lib/web_ui/dev/test_platform.dart

@@ -22,6 +22,8 @@ import 'package:shelf_web_socket/shelf_web_socket.dart';
 import 'package:shelf_packages_handler/shelf_packages_handler.dart';
 import 'package:stream_channel/stream_channel.dart';
 import 'package:web_socket_channel/web_socket_channel.dart';
+import 'package:web_test_utils/goldens.dart';
+import 'package:web_test_utils/image_compare.dart';
 
 import 'package:test_api/src/backend/runtime.dart'; // ignore: implementation_imports
 import 'package:test_api/src/backend/suite_platform.dart'; // ignore: implementation_imports
@@ -39,7 +41,6 @@ import 'package:test_core/src/runner/configuration.dart'; // ignore: implementat
 import 'browser.dart';
 import 'common.dart';
 import 'environment.dart' as env;
-import 'goldens.dart';
 import 'screenshot_manager.dart';
 import 'supported_browsers.dart';
 
@@ -197,17 +198,6 @@ class BrowserPlatform extends PlatformPlugin {
         'golden_files',
       );
     } else {
-      // On LUCI MacOS bots the goldens are fetched by the recipe code.
-      // Fetch the goldens if:
-      // - Tests are running on a local machine.
-      // - Tests are running on an OS other than macOS.
-      if (!isLuci || !Platform.isMacOS) {
-        await fetchGoldens();
-      } else {
-        if (!env.environment.webUiGoldensRepositoryDirectory.existsSync()) {
-          throw Exception('The goldens directory must have been copied');
-        }
-      }
       goldensDirectory = p.join(
         env.environment.webUiGoldensRepositoryDirectory.path,
         'engine',
@@ -215,19 +205,6 @@ class BrowserPlatform extends PlatformPlugin {
       );
     }
 
-    // Bail out fast if golden doesn't exist, and user doesn't want to create it.
-    final File file = File(p.join(
-      goldensDirectory,
-      filename,
-    ));
-    if (!file.existsSync() && !write) {
-      return '''
-Golden file $filename does not exist on path ${file.absolute.path}
-
-To automatically create this file call matchGoldenFile('$filename', write: true).
-''';
-    }
-
     final Rectangle regionAsRectange = Rectangle(
       region['x'] as num,
       region['y'] as num,
@@ -238,115 +215,14 @@ To automatically create this file call matchGoldenFile('$filename', write: true)
     // Take screenshot.
     final Image screenshot = await _screenshotManager.capture(regionAsRectange);
 
-    if (write) {
-      // Don't even bother with the comparison, just write and return
-      print('Updating screenshot golden: $file');
-      file.writeAsBytesSync(encodePng(screenshot), flush: true);
-      if (doUpdateScreenshotGoldens) {
-        // Do not fail tests when bulk-updating screenshot goldens.
-        return 'OK';
-      } else {
-        return 'Golden file $filename was updated. You can remove "write: true" in the call to matchGoldenFile.';
-      }
-    }
-
-    // Compare screenshots.
-    ImageDiff diff = ImageDiff(
-      golden: decodeNamedImage(file.readAsBytesSync(), filename),
-      other: screenshot,
-      pixelComparison: pixelComparison,
-    );
-
-    if (diff.rate > 0) {
-      final String testResultsPath =
-          env.environment.webUiTestResultsDirectory.path;
-      final String basename = p.basenameWithoutExtension(file.path);
-
-      final File actualFile =
-          File(p.join(testResultsPath, '$basename.actual.png'));
-      actualFile.writeAsBytesSync(encodePng(screenshot), flush: true);
-
-      final File diffFile = File(p.join(testResultsPath, '$basename.diff.png'));
-      diffFile.writeAsBytesSync(encodePng(diff.diff), flush: true);
-
-      final File expectedFile =
-          File(p.join(testResultsPath, '$basename.expected.png'));
-      file.copySync(expectedFile.path);
-
-      final File reportFile =
-          File(p.join(testResultsPath, '$basename.report.html'));
-      reportFile.writeAsStringSync('''
-Golden file $filename did not match the image generated by the test.
-
-<table>
-  <tr>
-    <th>Expected</th>
-    <th>Diff</th>
-    <th>Actual</th>
-  </tr>
-  <tr>
-    <td>
-      <img src="$basename.expected.png">
-    </td>
-    <td>
-      <img src="$basename.diff.png">
-    </td>
-    <td>
-      <img src="$basename.actual.png">
-    </td>
-  </tr>
-</table>
-''');
-
-      final StringBuffer message = StringBuffer();
-      message.writeln(
-          'Golden file $filename did not match the image generated by the test.');
-      message.writeln(getPrintableDiffFilesInfo(diff.rate, maxDiffRateFailure));
-      message
-          .writeln('You can view the test report in your browser by opening:');
-
-      // Cirrus cannot serve HTML pages generated by build jobs, so we
-      // archive all the files so that they can be downloaded and inspected
-      // locally.
-      if (isCirrus) {
-        final String taskId = Platform.environment['CIRRUS_TASK_ID'];
-        final String baseArtifactsUrl =
-            'https://api.cirrus-ci.com/v1/artifact/task/$taskId/web_engine_test/test_results';
-        final String cirrusReportUrl = '$baseArtifactsUrl/$basename.report.zip';
-        message.writeln(cirrusReportUrl);
-
-        await Process.run(
-          'zip',
-          <String>[
-            '$basename.report.zip',
-            '$basename.report.html',
-            '$basename.expected.png',
-            '$basename.diff.png',
-            '$basename.actual.png',
-          ],
-          workingDirectory: testResultsPath,
-        );
-      } else {
-        final String localReportPath = '$testResultsPath/$basename.report.html';
-        message.writeln(localReportPath);
-      }
-
-      message.writeln(
-          'To update the golden file call matchGoldenFile(\'$filename\', write: true).');
-      message.writeln('Golden file: ${expectedFile.path}');
-      message.writeln('Actual file: ${actualFile.path}');
-
-      if (diff.rate < maxDiffRateFailure) {
-        // Issue a warning but do not fail the test.
-        print('WARNING:');
-        print(message);
-        return 'OK';
-      } else {
-        // Fail test
-        return '$message';
-      }
-    }
-    return 'OK';
+    return compareImage(
+        screenshot,
+        doUpdateScreenshotGoldens,
+        filename,
+        pixelComparison,
+        maxDiffRateFailure,
+        goldensDirectory: goldensDirectory,
+        write: write);
   }
 
   /// A handler that serves wrapper files used to bootstrap tests.

lib/web_ui/dev/test_runner.dart

@@ -15,6 +15,7 @@ import 'package:test_core/src/runner/hack_register_platform.dart'
 import 'package:test_api/src/backend/runtime.dart'; // ignore: implementation_imports
 import 'package:test_core/src/executable.dart'
     as test; // ignore: implementation_imports
+import 'package:web_test_utils/goldens.dart';
 
 import 'common.dart';
 import 'environment.dart';
@@ -157,7 +158,10 @@ class TestCommand extends Command<bool> with ArgUtils {
   }
 
   Future<bool> runIntegrationTests() async {
-    return IntegrationTestsManager(browser, useSystemFlutter).runTests();
+    await _prepare();
+    return IntegrationTestsManager(
+            browser, useSystemFlutter, doUpdateScreenshotGoldens)
+        .runTests();
   }
 
   Future<bool> runUnitTests() async {
@@ -189,6 +193,15 @@ class TestCommand extends Command<bool> with ArgUtils {
     }
     environment.webUiTestResultsDirectory.createSync(recursive: true);
 
+    // If screenshot tests are available, fetch the screenshot goldens.
+    if (isScreenshotTestsAvailable) {
+      print('screenshot tests available');
+      final GoldensRepoFetcher goldensRepoFetcher = GoldensRepoFetcher(
+          environment.webUiGoldensRepositoryDirectory,
+          path.join(environment.webUiDevDir.path, 'goldens_lock.yaml'));
+      await goldensRepoFetcher.fetch();
+    }
+
     // In order to run iOS Safari unit tests we need to make sure iOS Simulator
     // is booted.
     if (isSafariIOS) {
@@ -371,6 +384,15 @@ class TestCommand extends Command<bool> with ArgUtils {
       isFirefoxIntegrationTestAvailable ||
       isSafariIntegrationTestAvailable;
 
+  // Whether the tests will do screenshot testing.
+  bool get isScreenshotTestsAvailable =>
+      isIntegrationTestsAvailable || isUnitTestsScreenshotsAvailable;
+
+  // For unit tests screenshot tests and smoke tests only run on:
+  // "Chrome/iOS" for LUCI/local.
+  bool get isUnitTestsScreenshotsAvailable =>
+      isChrome && (io.Platform.isLinux || !isLuci) || isSafariIOS;
+
   /// Use system flutter instead of cloning the repository.
   ///
   /// Read the flag help for more details. Uses PATH to locate flutter.
@@ -397,13 +419,7 @@ class TestCommand extends Command<bool> with ArgUtils {
       'test',
     ));
 
-    // Screenshot tests and smoke tests only run on: "Chrome/iOS Safari"
-    // locally and on LUCI. They are not available on Windows bots:
-    // TODO: https://github.com/flutter/flutter/issues/63710
-    if ((isChrome && isLuci && io.Platform.isLinux) ||
-        ((isChrome || isSafariIOS) && !isLuci) ||
-        (isSafariIOS && isLuci)) {
-      print('INFO: Also running the screenshot tests.');
+    if (isUnitTestsScreenshotsAvailable) {
       // Separate screenshot tests from unit-tests. Screenshot tests must run
       // one at a time. Otherwise, they will end up screenshotting each other.
       // This is not an issue for unit-tests.
@@ -621,7 +637,8 @@ class TestCommand extends Command<bool> with ArgUtils {
 
   /// Runs a batch of tests.
   ///
-  /// Unless [expectFailure] is set to false, sets [io.exitCode] to a non-zero value if any tests fail.
+  /// Unless [expectFailure] is set to false, sets [io.exitCode] to a non-zero
+  /// value if any tests fail.
   Future<void> _runTestBatch(
     List<FilePath> testFiles, {
     @required int concurrency,
@@ -644,7 +661,8 @@ class TestCommand extends Command<bool> with ArgUtils {
       return BrowserPlatform.start(
         browser,
         root: io.Directory.current.path,
-        // It doesn't make sense to update a screenshot for a test that is expected to fail.
+        // It doesn't make sense to update a screenshot for a test that is
+        // expected to fail.
         doUpdateScreenshotGoldens: !expectFailure && doUpdateScreenshotGoldens,
       );
     });

lib/web_ui/pubspec.yaml

@@ -14,7 +14,7 @@ dev_dependencies:
   image: 2.1.13
   js: 0.6.1+1
   mockito: 4.1.1
-  path: 1.7.0
+  path: 1.8.0-nullsafety.1
   test: 1.14.3
   quiver: 2.1.3
   build_resolvers: 1.3.10
@@ -23,6 +23,8 @@ dev_dependencies:
   build_web_compilers: 2.11.0
   yaml: 2.2.1
   watcher: 0.9.7+15
+  web_test_utils:
+    path: ../../web_sdk/web_test_utils
   web_engine_tester:
     path: ../../web_sdk/web_engine_tester
   simulators:

web_sdk/web_test_utils/lib/environment.dart

+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// @dart = 2.6
+import 'dart:io' as io;
+import 'package:path/path.dart' as pathlib;
+
+import 'exceptions.dart';
+
+/// Contains various environment variables, such as common file paths and command-line options.
+Environment get environment {
+  _environment ??= Environment();
+  return _environment;
+}
+
+Environment _environment;
+
+/// Contains various environment variables, such as common file paths and command-line options.
+class Environment {
+  factory Environment() {
+    final io.File self = io.File.fromUri(io.Platform.script);
+    final io.Directory engineSrcDir = self.parent.parent.parent.parent.parent;
+    return _prepareEnvironmentFromEngineDir(self, engineSrcDir);
+  }
+
+  factory Environment.forIntegrationTests() {
+    final io.File self = io.File.fromUri(io.Platform.script);
+    final io.Directory engineSrcDir =
+        self.parent.parent.parent.parent.parent.parent;
+    return _prepareEnvironmentFromEngineDir(self, engineSrcDir);
+  }
+
+  static Environment _prepareEnvironmentFromEngineDir(
+      io.File self, io.Directory engineSrcDir) {
+    final io.Directory engineToolsDir =
+        io.Directory(pathlib.join(engineSrcDir.path, 'flutter', 'tools'));
+    final io.Directory outDir =
+        io.Directory(pathlib.join(engineSrcDir.path, 'out'));
+    final io.Directory hostDebugUnoptDir =
+        io.Directory(pathlib.join(outDir.path, 'host_debug_unopt'));
+    final io.Directory dartSdkDir =
+        io.Directory(pathlib.join(hostDebugUnoptDir.path, 'dart-sdk'));
+    final io.Directory webUiRootDir = io.Directory(
+        pathlib.join(engineSrcDir.path, 'flutter', 'lib', 'web_ui'));
+    final io.Directory integrationTestsDir = io.Directory(
+        pathlib.join(engineSrcDir.path, 'flutter', 'e2etests', 'web'));
+
+    for (io.Directory expectedDirectory in <io.Directory>[
+      engineSrcDir,
+      outDir,
+      hostDebugUnoptDir,
+      dartSdkDir,
+      webUiRootDir
+    ]) {
+      if (!expectedDirectory.existsSync()) {
+        throw ToolException('$expectedDirectory does not exist.');
+      }
+    }
+
+    return Environment._(
+      self: self,
+      webUiRootDir: webUiRootDir,
+      engineSrcDir: engineSrcDir,
+      engineToolsDir: engineToolsDir,
+      integrationTestsDir: integrationTestsDir,
+      outDir: outDir,
+      hostDebugUnoptDir: hostDebugUnoptDir,
+      dartSdkDir: dartSdkDir,
+    );
+  }
+
+  Environment._({
+    this.self,
+    this.webUiRootDir,
+    this.engineSrcDir,
+    this.engineToolsDir,
+    this.integrationTestsDir,
+    this.outDir,
+    this.hostDebugUnoptDir,
+    this.dartSdkDir,
+  });
+
+  /// The Dart script that's currently running.
+  final io.File self;
+
+  /// Path to the "web_ui" package sources.
+  final io.Directory webUiRootDir;
+
+  /// Path to the engine's "src" directory.
+  final io.Directory engineSrcDir;
+
+  /// Path to the engine's "tools" directory.
+  final io.Directory engineToolsDir;
+
+  /// Path to the web integration tests.
+  final io.Directory integrationTestsDir;
+
+  /// Path to the engine's "out" directory.
+  ///
+  /// This is where you'll find the ninja output, such as the Dart SDK.
+  final io.Directory outDir;
+
+  /// The "host_debug_unopt" build of the Dart SDK.
+  final io.Directory hostDebugUnoptDir;
+
+  /// The root of the Dart SDK.
+  final io.Directory dartSdkDir;
+
+  /// The "dart" executable file.
+  String get dartExecutable => pathlib.join(dartSdkDir.path, 'bin', 'dart');
+
+  /// The "pub" executable file.
+  String get pubExecutable => pathlib.join(dartSdkDir.path, 'bin', 'pub');
+
+  /// The "dart2js" executable file.
+  String get dart2jsExecutable =>
+      pathlib.join(dartSdkDir.path, 'bin', 'dart2js');
+
+  /// Path to where github.com/flutter/engine is checked out inside the engine workspace.
+  io.Directory get flutterDirectory =>
+      io.Directory(pathlib.join(engineSrcDir.path, 'flutter'));
+  io.Directory get webSdkRootDir => io.Directory(pathlib.join(
+        flutterDirectory.path,
+        'web_sdk',
+      ));
+
+  /// Path to the "web_engine_tester" package.
+  io.Directory get webEngineTesterRootDir => io.Directory(pathlib.join(
+        webSdkRootDir.path,
+        'web_engine_tester',
+      ));
+
+  /// Path to the "build" directory, generated by "package:build_runner".
+  ///
+  /// This is where compiled output goes.
+  io.Directory get webUiBuildDir => io.Directory(pathlib.join(
+        webUiRootDir.path,
+        'build',
+      ));
+
+  /// Path to the ".dart_tool" directory, generated by various Dart tools.
+  io.Directory get webUiDartToolDir => io.Directory(pathlib.join(
+        webUiRootDir.path,
+        '.dart_tool',
+      ));
+
+  /// Path to the ".dart_tool" directory living under `engine/src/flutter`.
+  ///
+  /// This is a designated area for tool downloads which can be used by
+  /// multiple platforms. For exampe: Flutter repo for e2e tests.
+  io.Directory get engineDartToolDir => io.Directory(pathlib.join(
+        engineSrcDir.path,
+        'flutter',
+        '.dart_tool',
+      ));
+
+  /// Path to the "dev" directory containing engine developer tools and
+  /// configuration files.
+  io.Directory get webUiDevDir => io.Directory(pathlib.join(
+        webUiRootDir.path,
+        'dev',
+      ));
+
+  /// Path to the "test" directory containing web engine tests.
+  io.Directory get webUiTestDir => io.Directory(pathlib.join(
+        webUiRootDir.path,
+        'test',
+      ));
+
+  /// Path to the "lib" directory containing web engine code.
+  io.Directory get webUiLibDir => io.Directory(pathlib.join(
+        webUiRootDir.path,
+        'lib',
+      ));
+
+  /// Path to the clone of the flutter/goldens repository.
+  io.Directory get webUiGoldensRepositoryDirectory => io.Directory(pathlib.join(
+        webUiDartToolDir.path,
+        'goldens',
+      ));
+
+  /// Directory to add test results which would later be uploaded to a gcs
+  /// bucket by LUCI.
+  io.Directory get webUiTestResultsDirectory => io.Directory(pathlib.join(
+        webUiDartToolDir.path,
+        'test_results',
+      ));
+
+  /// Path to the screenshots taken by iOS simulator.
+  io.Directory get webUiSimulatorScreenshotsDirectory =>
+      io.Directory(pathlib.join(
+        webUiDartToolDir.path,
+        'ios_screenshots',
+      ));
+
+  /// Path to the script that clones the Flutter repo.
+  io.File get cloneFlutterScript => io.File(pathlib.join(
+        engineToolsDir.path,
+        'clone_flutter.sh',
+      ));
+
+  /// Path to flutter.
+  ///
+  /// For example, this can be used to run `flutter pub get`.
+  ///
+  /// Only use [cloneFlutterScript] to clone flutter to the engine build.
+  io.File get flutterCommand => io.File(pathlib.join(
+        engineDartToolDir.path,
+        'flutter',
+        'bin',
+        'flutter',
+      ));
+}

web_sdk/web_test_utils/lib/exceptions.dart

+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+class BrowserInstallerException implements Exception {
+  BrowserInstallerException(this.message);
+
+  final String message;
+
+  @override
+  String toString() => message;
+}
+
+class DriverException implements Exception {
+  DriverException(this.message);
+
+  final String message;
+
+  @override
+  String toString() => message;
+}
+
+class ToolException implements Exception {
+  ToolException(this.message);
+
+  final String message;
+
+  @override
+  String toString() => message;
+}

lib/web_ui/dev/goldens.dart → web_sdk/web_test_utils/lib/goldens.dart

@@ -6,10 +6,8 @@ import 'dart:io' as io;
 import 'package:image/image.dart';
 import 'package:meta/meta.dart';
 import 'package:path/path.dart' as path;
-import 'package:yaml/yaml.dart';
 
-import 'environment.dart';
-import 'utils.dart';
+import 'package:yaml/yaml.dart';
 
 /// How to compares pixels within the image.
 ///
@@ -30,14 +28,14 @@ void main(List<String> args) {
   final io.File fileB = io.File(args[1]);
   final Image imageA = decodeNamedImage(fileA.readAsBytesSync(), 'a.png');
   final Image imageB = decodeNamedImage(fileB.readAsBytesSync(), 'b.png');
-  final ImageDiff diff = ImageDiff(golden: imageA, other: imageB, pixelComparison: PixelComparison.fuzzy);
+  final ImageDiff diff = ImageDiff(
+      golden: imageA, other: imageB, pixelComparison: PixelComparison.fuzzy);
   print('Diff: ${(diff.rate * 100).toStringAsFixed(4)}%');
 }
 
 /// This class encapsulates visually diffing an Image with any other.
 /// Both images need to be the exact same size.
 class ImageDiff {
-
   /// The image to match
   final Image golden;
 
@@ -59,6 +57,8 @@ class ImageDiff {
   /// This gets set to 1 (100% difference) when golden and other aren't the same size.
   double get rate => _wrongPixels / _pixelCount;
 
+  /// Image diff constructor which requires two [Image]s to compare and
+  /// [PixelComparison] algorithm.
   ImageDiff({
     @required this.golden,
     @required this.other,
@@ -72,8 +72,8 @@ class ImageDiff {
 
   /// That would be the distance between black and white.
   static final double _maxTheoreticalColorDistance = Color.distance(
-    <num>[255, 255, 255],  // white
-    <num>[0, 0, 0],  // black
+    <num>[255, 255, 255], // white
+    <num>[0, 0, 0], // black
     false,
   ).toDouble();
 
@@ -121,11 +121,9 @@ class ImageDiff {
           _reflectedPixel(image, x - 1, y - 1),
           _reflectedPixel(image, x - 1, y),
           _reflectedPixel(image, x - 1, y + 1),
-
           _reflectedPixel(image, x, y - 1),
           _reflectedPixel(image, x, y),
           _reflectedPixel(image, x, y + 1),
-
           _reflectedPixel(image, x + 1, y - 1),
           _reflectedPixel(image, x + 1, y),
           _reflectedPixel(image, x + 1, y + 1),
@@ -148,25 +146,30 @@ class ImageDiff {
   }
 
   void _computeDiff() {
-    int goldenWidth = golden.width;
-    int goldenHeight = golden.height;
+    final int goldenWidth = golden.width;
+    final int goldenHeight = golden.height;
 
     _pixelCount = goldenWidth * goldenHeight;
     diff = Image(goldenWidth, goldenHeight);
 
     if (goldenWidth == other.width && goldenHeight == other.height) {
-      for(int y = 0; y < goldenHeight; y++) {
+      for (int y = 0; y < goldenHeight; y++) {
         for (int x = 0; x < goldenWidth; x++) {
-          final bool isExactlySame = golden.getPixel(x, y) == other.getPixel(x, y);
+          final bool isExactlySame =
+              golden.getPixel(x, y) == other.getPixel(x, y);
           final List<int> goldenPixel = _getPixelRgbForComparison(golden, x, y);
           final List<int> otherPixel = _getPixelRgbForComparison(other, x, y);
-          final double colorDistance = Color.distance(goldenPixel, otherPixel, false) / _maxTheoreticalColorDistance;
+          final double colorDistance =
+              Color.distance(goldenPixel, otherPixel, false) /
+                  _maxTheoreticalColorDistance;
           final bool isFuzzySame = colorDistance < _kColorDistanceThreshold;
           if (isExactlySame || isFuzzySame) {
             diff.setPixel(x, y, _colorOk);
           } else {
-            final int goldenLuminance = getLuminanceRgb(goldenPixel[0], goldenPixel[1], goldenPixel[2]);
-            final int otherLuminance = getLuminanceRgb(otherPixel[0], otherPixel[1], otherPixel[2]);
+            final int goldenLuminance =
+                getLuminanceRgb(goldenPixel[0], goldenPixel[1], goldenPixel[2]);
+            final int otherLuminance =
+                getLuminanceRgb(otherPixel[0], otherPixel[1], otherPixel[2]);
             if (goldenLuminance < otherLuminance) {
               diff.setPixel(x, y, _colorExpectedPixel);
             } else {
@@ -183,26 +186,31 @@ class ImageDiff {
   }
 }
 
-// Returns text explaining pixel difference rate.
+/// Returns text explaining pixel difference rate.
 String getPrintableDiffFilesInfo(double diffRate, double maxRate) =>
-  '(${((diffRate) * 100).toStringAsFixed(4)}% of pixels were different. '
-  'Maximum allowed rate is: ${(maxRate * 100).toStringAsFixed(4)}%).';
+    '(${((diffRate) * 100).toStringAsFixed(4)}% of pixels were different. '
+    'Maximum allowed rate is: ${(maxRate * 100).toStringAsFixed(4)}%).';
 
-/// Fetches golden files from github.com/flutter/goldens, cloning the repository if necessary.
+/// Downloads the repository that stores the golden files.
 ///
-/// The repository is cloned into web_ui/.dart_tool.
-Future<void> fetchGoldens() async {
-  await _GoldensRepoFetcher().fetch();
-}
-
-class _GoldensRepoFetcher {
+/// Reads the url of the repo and `commit no` to sync to, from
+/// `goldens_lock.yaml`.
+class GoldensRepoFetcher {
   String _repository;
   String _revision;
+  final io.Directory _webUiGoldensRepositoryDirectory;
+  final String _lockFilePath;
 
+  /// Constructor that takes directory to download the repository and
+  /// file with goldens repo information.
+  GoldensRepoFetcher(this._webUiGoldensRepositoryDirectory, this._lockFilePath);
+
+  /// Fetches golden files from github.com/flutter/goldens, cloning the
+  /// repository if necessary.
+  ///
+  /// The repository is cloned into web_ui/.dart_tool.
   Future<void> fetch() async {
-    final io.File lockFile = io.File(
-      path.join(environment.webUiDevDir.path, 'goldens_lock.yaml')
-    );
+    final io.File lockFile = io.File(path.join(_lockFilePath));
     final YamlMap lock = loadYaml(lockFile.readAsStringSync()) as YamlMap;
     _repository = lock['repository'] as String;
     _revision = lock['revision'] as String;
@@ -214,40 +222,32 @@ class _GoldensRepoFetcher {
 
     print('Fetching $_repository@$_revision');
 
-    if (!environment.webUiGoldensRepositoryDirectory.existsSync()) {
-      environment.webUiGoldensRepositoryDirectory.createSync(recursive: true);
-      await runProcess(
-        'git',
+    if (!_webUiGoldensRepositoryDirectory.existsSync()) {
+      _webUiGoldensRepositoryDirectory.createSync(recursive: true);
+      await _runGit(
         <String>['init'],
-        workingDirectory: environment.webUiGoldensRepositoryDirectory.path,
-        mustSucceed: true,
+        _webUiGoldensRepositoryDirectory.path,
       );
-      await runProcess(
-        'git',
+
+      await _runGit(
         <String>['remote', 'add', 'origin', _repository],
-        workingDirectory: environment.webUiGoldensRepositoryDirectory.path,
-        mustSucceed: true,
+        _webUiGoldensRepositoryDirectory.path,
       );
     }
 
-    await runProcess(
-      'git',
+    await _runGit(
       <String>['fetch', 'origin', 'master'],
-      workingDirectory: environment.webUiGoldensRepositoryDirectory.path,
-      mustSucceed: true,
+      _webUiGoldensRepositoryDirectory.path,
     );
-    await runProcess(
-      'git',
+    await _runGit(
       <String>['checkout', _revision],
-      workingDirectory: environment.webUiGoldensRepositoryDirectory.path,
-      mustSucceed: true,
+      _webUiGoldensRepositoryDirectory.path,
     );
   }
 
   Future<String> _getLocalRevision() async {
-    final io.File head = io.File(path.join(
-      environment.webUiGoldensRepositoryDirectory.path, '.git', 'HEAD'
-    ));
+    final io.File head = io.File(
+        path.join(_webUiGoldensRepositoryDirectory.path, '.git', 'HEAD'));
 
     if (!head.existsSync()) {
       return null;
@@ -255,4 +255,26 @@ class _GoldensRepoFetcher {
 
     return head.readAsStringSync().trim();
   }
+
+  /// Runs `git` with given arguments.
+  Future<void> _runGit(
+    List<String> arguments,
+    String workingDirectory,
+  ) async {
+    final io.Process process = await io.Process.start(
+      'git',
+      arguments,
+      workingDirectory: workingDirectory,
+      // Running the process in a system shell for Windows. Otherwise
+      // the process is not able to get Dart from path.
+      runInShell: io.Platform.isWindows,
+      mode: io.ProcessStartMode.inheritStdio,
+    );
+    final int exitCode = await process.exitCode;
+    if (exitCode != 0) {
+      throw Exception('Git command failed with arguments $arguments on '
+          'workingDirectory: $workingDirectory resulting with exitCode: '
+          '$exitCode');
+    }
+  }
 }

web_sdk/web_test_utils/lib/image_compare.dart

+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:image/image.dart';
+import 'package:path/path.dart' as p;
+
+import 'environment.dart';
+import 'goldens.dart';
+
+/// Compares a screenshot taken through a test with it's golden.
+///
+/// Used by Flutter Web Engine unit tests and the integration tests.
+///
+/// Returns the results of the tests as `String`. When tests passes the result
+/// is simply `OK`, however when they fail it contains a detailed explanation
+/// on which files are compared, their absolute locations and an HTML page
+/// that the developer can see the comparison.
+String compareImage(
+  Image screenshot,
+  bool doUpdateScreenshotGoldens,
+  String filename,
+  PixelComparison pixelComparison,
+  double maxDiffRateFailure, {
+  String goldensDirectory = '',
+  bool forIntegrationTests = false,
+  bool write = false,
+}) {
+  final Environment environment =
+      forIntegrationTests ? Environment.forIntegrationTests() : Environment();
+  if (goldensDirectory.isEmpty) {
+    goldensDirectory = p.join(
+      environment.webUiGoldensRepositoryDirectory.path,
+      'engine',
+      'web',
+    );
+  }
+  // Bail out fast if golden doesn't exist, and user doesn't want to create it.
+  final File file = File(p.join(
+    goldensDirectory,
+    filename,
+  ));
+  if (!file.existsSync() && !write) {
+    return '''
+Golden file $filename does not exist.
+
+To automatically create this file call matchGoldenFile('$filename', write: true).
+''';
+  }
+  if (write) {
+    // Don't even bother with the comparison, just write and return
+    print('Updating screenshot golden: $file');
+    file.writeAsBytesSync(encodePng(screenshot), flush: true);
+    if (doUpdateScreenshotGoldens) {
+      // Do not fail tests when bulk-updating screenshot goldens.
+      return 'OK';
+    } else {
+      return 'Golden file $filename was updated. You can remove "write: true" '
+          'in the call to matchGoldenFile.';
+    }
+  }
+
+  final Image golden = decodeNamedImage(file.readAsBytesSync(), filename);
+
+  // Compare screenshots.
+  final ImageDiff diff = ImageDiff(
+    golden: golden,
+    other: screenshot,
+    pixelComparison: pixelComparison,
+  );
+
+  if (diff.rate > 0) {
+    final String testResultsPath = environment.webUiTestResultsDirectory.path;
+    Directory(testResultsPath).createSync(recursive: true);
+    final String basename = p.basenameWithoutExtension(file.path);
+
+    final File actualFile =
+        File(p.join(testResultsPath, '$basename.actual.png'));
+    actualFile.writeAsBytesSync(encodePng(screenshot), flush: true);
+
+    final File diffFile = File(p.join(testResultsPath, '$basename.diff.png'));
+    diffFile.writeAsBytesSync(encodePng(diff.diff), flush: true);
+
+    final File expectedFile =
+        File(p.join(testResultsPath, '$basename.expected.png'));
+    file.copySync(expectedFile.path);
+
+    final File reportFile =
+        File(p.join(testResultsPath, '$basename.report.html'));
+    reportFile.writeAsStringSync('''
+Golden file $filename did not match the image generated by the test.
+
+<table>
+  <tr>
+    <th>Expected</th>
+    <th>Diff</th>
+    <th>Actual</th>
+  </tr>
+  <tr>
+    <td>
+      <img src="$basename.expected.png">
+    </td>
+    <td>
+      <img src="$basename.diff.png">
+    </td>
+    <td>
+      <img src="$basename.actual.png">
+    </td>
+  </tr>
+</table>
+''');
+
+    final StringBuffer message = StringBuffer();
+    message.writeln(
+        'Golden file $filename did not match the image generated by the test.');
+    message.writeln(getPrintableDiffFilesInfo(diff.rate, maxDiffRateFailure));
+    message.writeln('You can view the test report in your browser by opening:');
+
+    final String localReportPath = '$testResultsPath/$basename.report.html';
+    message.writeln(localReportPath);
+
+    message.writeln(
+        'To update the golden file call matchGoldenFile(\'$filename\', write: '
+        'true).');
+    message.writeln('Golden file: ${expectedFile.path}');
+    message.writeln('Actual file: ${actualFile.path}');
+
+    if (diff.rate < maxDiffRateFailure) {
+      // Issue a warning but do not fail the test.
+      print('WARNING:');
+      print(message);
+      return 'OK';
+    } else {
+      // Fail test
+      return '$message';
+    }
+  }
+  return 'OK';
+}

web_sdk/web_test_utils/pubspec.yaml

+name: web_test_utils
+
+environment:
+  sdk: ">=2.2.0 <3.0.0"
+
+dependencies:
+  path: 1.8.0-nullsafety.1
+  image: 2.1.13
+  js: 0.6.1+1
+  yaml: 2.2.1