From 3b2da6be9f32e63c00252b31b3c9da1d6eeda60c Mon Sep 17 00:00:00 2001 From: nturgut Date: Tue, 13 Oct 2020 19:26:34 -0700 Subject: [PATCH] [web] enabling firefox screenshot tests. adding to documentation (#21807) * enabling firefox screenshot tests. adding to documentation * test with correct goldens * update goldens SHA * change the goldens with flutter/goldens repo * do not run preparation step twice. this will cause test_results to be deleted --- .../web/regular_integration_tests/README.md | 8 +++-- .../lib/screenshot_support.dart | 35 ++++++++----------- lib/web_ui/dev/goldens_lock.yaml | 2 +- lib/web_ui/dev/test_runner.dart | 12 ++++++- 4 files changed, 33 insertions(+), 24 deletions(-) diff --git a/e2etests/web/regular_integration_tests/README.md b/e2etests/web/regular_integration_tests/README.md index 2e0aa9616..5e3501564 100644 --- a/e2etests/web/regular_integration_tests/README.md +++ b/e2etests/web/regular_integration_tests/README.md @@ -55,7 +55,7 @@ Future main() async { In order to run the tests follow these steps: -1. You can use two different approaches, using [felt](https://github.com/flutter/engine/blob/master/lib/web_ui/dev/README.md) tool will run all the tests, an update all the goldens. For running individual tests, we need to set UPDATE_GOLDENS environment variable. +1. You can use two different approaches, using [felt](https://github.com/flutter/engine/blob/master/lib/web_ui/dev/README.md) tool will run all the tests, hence update all the goldens. For running individual tests, we need to set UPDATE_GOLDENS environment variable. Screenshots are saved differently per browser, therefore do not forget to also update the screenshots for other browsers. Note that, LUCI is only running screenshot testing for integration tests on Firefox and Chrome. ``` felt test --integration-tests-only --update-screenshot-goldens @@ -65,6 +65,10 @@ felt test --integration-tests-only --update-screenshot-goldens UPDATE_GOLDENS=true flutter drive -v --target=test_driver/text_editing_integration.dart -d web-server --release --local-engine=host_debug_unopt ``` -2. The golden will be under `engine/src/flutter/lib/web_ui/.dart_tool/goldens/engine/web/` directory, you should create a PR for that file and merge it to `flutter/goldens`. +``` +UPDATE_GOLDENS=true flutter drive -v --target=test_driver/text_editing_integration.dart -d web-server --release --local-engine=host_debug_unopt --browser-name=firefox +``` + +2. The golden will be under `engine/src/flutter/lib/web_ui/.dart_tool/goldens/engine/web/` directory, you should create a PR for that file and merge it to `flutter/goldens`. For each browser the browser name would be appended to the end of the golden file such as: `screenshot_name-chrome.png` or `screenshot_name-firefox.png` 3. Get the commit SHA and replace the `revision` in this file: `engine/src/flutter/lib/web_ui/dev/goldens_lock.yaml` diff --git a/e2etests/web/regular_integration_tests/lib/screenshot_support.dart b/e2etests/web/regular_integration_tests/lib/screenshot_support.dart index 01d97067d..bfdf34ff0 100644 --- a/e2etests/web/regular_integration_tests/lib/screenshot_support.dart +++ b/e2etests/web/regular_integration_tests/lib/screenshot_support.dart @@ -53,8 +53,7 @@ Future runTestWithScreenshots( bool updateGoldens = false; // We are using an environment variable instead of an argument, since // this code is not invoked from the shell but from the `flutter drive` - // tool itself. Therefore we do not have control on the command line - // arguments. + // tool itself, we do not have control on the command line arguments. // Please read the README, further info on how to update the goldens. final String updateGoldensFlag = io.Platform.environment['UPDATE_GOLDENS']; // Validate if the environment variable is set correctly. @@ -71,25 +70,21 @@ Future runTestWithScreenshots( test.integrationDriver( driver: driver, onScreenshot: (String screenshotName, List screenshotBytes) async { - if (browser == 'chrome') { - final Image screenshot = decodePng(screenshotBytes); - final String result = compareImage( - screenshot, - updateGoldens, - '$screenshotName-$browser.png', - PixelComparison.fuzzy, - diffRateFailure, - forIntegrationTests: true, - write: updateGoldens, - ); - if (result == 'OK') { - return true; - } else { - io.stderr.writeln('ERROR: $result'); - return false; - } - } else { + final Image screenshot = decodePng(screenshotBytes); + final String result = compareImage( + screenshot, + updateGoldens, + '$screenshotName-$browser.png', + PixelComparison.fuzzy, + diffRateFailure, + forIntegrationTests: true, + write: updateGoldens, + ); + if (result == 'OK') { return true; + } else { + io.stderr.writeln('ERROR: $result'); + return false; } }, ); diff --git a/lib/web_ui/dev/goldens_lock.yaml b/lib/web_ui/dev/goldens_lock.yaml index 579e6c508..66a641aed 100644 --- a/lib/web_ui/dev/goldens_lock.yaml +++ b/lib/web_ui/dev/goldens_lock.yaml @@ -1,2 +1,2 @@ repository: https://github.com/flutter/goldens.git -revision: da3fef0c0eb849dfbb14b09a088c5f7916677482 +revision: 672510dc52daa5b059081f6990582bccdb4ea48f diff --git a/lib/web_ui/dev/test_runner.dart b/lib/web_ui/dev/test_runner.dart index 74c8de6d8..9538a380e 100644 --- a/lib/web_ui/dev/test_runner.dart +++ b/lib/web_ui/dev/test_runner.dart @@ -103,6 +103,13 @@ class TestCommand extends Command with ArgUtils { /// How many dart2js build tasks are running at the same time. final Pool _pool = Pool(8); + /// Checks if test harness preparation (such as fetching the goldens, + /// creating test_results directory or starting ios-simulator) has been done. + /// + /// If unit tests already did these steps, integration tests do not have to + /// repeat them. + bool _testPreparationReady = false; + /// Check the flags to see what type of tests are requested. TestTypesRequested findTestType() { if (boolArg('unit-tests-only') && boolArg('integration-tests-only')) { @@ -158,7 +165,9 @@ class TestCommand extends Command with ArgUtils { } Future runIntegrationTests() async { - await _prepare(); + if(!_testPreparationReady) { + await _prepare(); + } return IntegrationTestsManager( browser, useSystemFlutter, doUpdateScreenshotGoldens) .runTests(); @@ -207,6 +216,7 @@ class TestCommand extends Command with ArgUtils { if (isSafariIOS) { await IosSafariArgParser.instance.initIosSimulator(); } + _testPreparationReady = true; } /// Builds all test targets that will be run. -- GitLab