Make the testing process on different platforms more consistent (#18660)

testing/ios/IosUnitTests/README.md

@@ -6,13 +6,12 @@ also run in LUCI builds.
 ## Running Tests
 
 ```sh
-./flutter/tools/gn --ios --simulator --unoptimized
-cd flutter/testing/ios/IosUnitTests
-./build_and_run_tests.sh
+testing/run_tests.py [--type=objc]
 ```
 
 After the `ios_flutter_test` target is built you can also run the tests inside
-of xcode with `IosUnitTests.xcodeproj`.
+of Xcode with `testing/ios/IosUnitTests/IosUnitTests.xcodeproj`. If you
+modify the test or under-test files, you'll have to run `run_tests.py` again.
 
 ## Adding Tests
 

testing/run_tests.py

@@ -258,6 +258,7 @@ def EnsureDebugUnoptSkyPackagesAreBuilt():
   RunCmd(ninja_command, cwd=buildroot_dir)
 
 def EnsureJavaTestsAreBuilt(android_out_dir):
+  """Builds the engine variant and the test jar containing the JUnit tests"""
   ninja_command = [
     'autoninja',
     '-C',
@@ -271,6 +272,8 @@ def EnsureJavaTestsAreBuilt(android_out_dir):
     RunCmd(ninja_command, cwd=buildroot_dir)
     return
 
+  assert android_out_dir != "out/android_debug_unopt", "%s doesn't exist. Run GN to generate the directory first" % android_out_dir
+
   # Otherwise prepare the directory first, then build the test.
   gn_command = [
     os.path.join(buildroot_dir, 'flutter', 'tools', 'gn'),
@@ -282,15 +285,53 @@ def EnsureJavaTestsAreBuilt(android_out_dir):
   RunCmd(gn_command, cwd=buildroot_dir)
   RunCmd(ninja_command, cwd=buildroot_dir)
 
+def EnsureIosTestsAreBuilt(ios_out_dir):
+  """Builds the engine variant and the test dylib containing the XCTests"""
+  ninja_command = [
+    'autoninja',
+    '-C',
+    ios_out_dir,
+    'ios_test_flutter'
+  ]
+
+  # Attempt running Ninja if the out directory exists.
+  # We don't want to blow away any custom GN args the caller may have already set.
+  if os.path.exists(ios_out_dir):
+    RunCmd(ninja_command, cwd=buildroot_dir)
+    return
+
+  assert ios_out_dir != "out/ios_debug_sim_unopt", "%s doesn't exist. Run GN to generate the directory first" % ios_out_dir
+
+  # Otherwise prepare the directory first, then build the test.
+  gn_command = [
+    os.path.join(buildroot_dir, 'flutter', 'tools', 'gn'),
+    '--ios',
+    '--unoptimized',
+    '--runtime-mode=debug',
+    '--no-lto',
+    '--simulator'
+  ]
+  RunCmd(gn_command, cwd=buildroot_dir)
+  RunCmd(ninja_command, cwd=buildroot_dir)
+
 def AssertExpectedJavaVersion():
+  """Checks that the user has Java 8 which is the supported Java version for Android"""
   EXPECTED_VERSION = '1.8'
   # `java -version` is output to stderr. https://bugs.java.com/bugdatabase/view_bug.do?bug_id=4380614
   version_output = subprocess.check_output(['java', '-version'], stderr=subprocess.STDOUT)
   match = bool(re.compile('version "%s' % EXPECTED_VERSION).search(version_output))
   message = "JUnit tests need to be run with Java %s. Check the `java -version` on your PATH." % EXPECTED_VERSION
-  assert True, message
+  assert match, message
+
+def AssertExpectedXcodeVersion():
+  """Checks that the user has a recent version of Xcode installed"""
+  EXPECTED_MAJOR_VERSION = '11'
+  version_output = subprocess.check_output(['xcodebuild', '-version'])
+  message = "Xcode must be installed to run the iOS embedding unit tests"
+  assert "Xcode %s." % EXPECTED_MAJOR_VERSION in version_output, message
 
 def RunJavaTests(filter, android_variant='android_debug_unopt'):
+  """Runs the Java JUnit unit tests for the Android embedding"""
   AssertExpectedJavaVersion()
   android_out_dir = os.path.join(out_dir, android_variant)
   EnsureJavaTestsAreBuilt(android_out_dir)
@@ -316,6 +357,26 @@ def RunJavaTests(filter, android_variant='android_debug_unopt'):
 
   RunCmd(command)
 
+def RunObjcTests(ios_variant='ios_debug_sim_unopt'):
+  """Runs Objective-C XCTest unit tests for the iOS embedding"""
+  AssertExpectedXcodeVersion()
+  ios_out_dir = os.path.join(out_dir, ios_variant)
+  EnsureIosTestsAreBuilt(ios_out_dir)
+
+  pretty = "cat" if subprocess.call(["which", "xcpretty"]) else "xcpretty"
+  ios_unit_test_dir = os.path.join(buildroot_dir, 'flutter', 'testing', 'ios', 'IosUnitTests')
+
+  command = [
+    'xcodebuild '
+    '-sdk iphonesimulator '
+    '-scheme IosUnitTests '
+    "-destination platform='iOS Simulator,name=iPhone 8' "
+    'test '
+    'FLUTTER_ENGINE=' + ios_variant +
+    ' | ' + pretty
+  ]
+  RunCmd(command, cwd=ios_unit_test_dir, shell=True)
+
 def RunDartTests(build_dir, filter, verbose_dart_snapshot):
   # This one is a bit messy. The pubspec.yaml at flutter/testing/dart/pubspec.yaml
   # has dependencies that are hardcoded to point to the sky packages at host_debug_unopt/
@@ -375,13 +436,16 @@ def main():
   parser.add_argument('--android-variant', dest='android_variant', action='store',
       default='android_debug_unopt',
       help='The engine build variant to run java tests for')
+  parser.add_argument('--ios-variant', dest='ios_variant', action='store',
+      default='ios_debug_sim_unopt',
+      help='The engine build variant to run objective-c tests for')
   parser.add_argument('--verbose-dart-snapshot', dest='verbose_dart_snapshot', action='store_true',
       default=False, help='Show extra dart snapshot logging.')
 
   args = parser.parse_args()
 
   if args.type == 'all':
-    types = ['engine', 'dart', 'benchmarks', 'java', 'font-subset']
+    types = ['engine', 'dart', 'benchmarks', 'java', 'objc', 'font-subset']
   else:
     types = args.type.split(',')
 
@@ -408,6 +472,10 @@ def main():
       java_filter = None
     RunJavaTests(java_filter, args.android_variant)
 
+  if 'objc' in types:
+    assert IsMac(), "iOS embedding tests can only be run on macOS."
+    RunObjcTests(args.ios_variant)
+
   # https://github.com/flutter/flutter/issues/36300
   if 'benchmarks' in types and not IsWindows():
     RunEngineBenchmarks(build_dir, engine_filter)

testing/scenario_app/README.md

 # Scenario App
 
-This folder contains a dart:ui application and scripts to compile it to JIT or
-AOT for exercising embedders.
+This folder contains e2e integration tests for the engine in conjunction with a
+fake dart:ui framework running in JIT or AOT.
 
 It intentionally has no dependencies on the Flutter framework or tooling, such
 that it should be buildable as a presubmit or postsubmit to the engine even in
 the face of changes to Dart or dart:ui that require upstream changes in the
 Flutter tooling.
 
+## Adding a New Scenario
+
+Create a new subclass of [Scenario](https://github.com/flutter/engine/blob/5d9509ae056b04c30295df27f201f31af9777842/testing/scenario_app/lib/src/scenario.dart#L9)
+and add it to the map in [main.dart](https://github.com/flutter/engine/blob/5d9509ae056b04c30295df27f201f31af9777842/testing/scenario_app/lib/main.dart#L17).
+For an example, see [animated_color_square.dart](https://github.com/flutter/engine/blob/5d9509ae056b04c30295df27f201f31af9777842/testing/scenario_app/lib/src/animated_color_square.dart#L15),
+which draws a continuously animating colored square that bounces off the sides
+of the viewport.
+
+Then set the scenario from the Android or iOS app by calling "set_scenario" on
+platform channel.
+
 ## Running for iOS
 
 ```sh
-cd ${ENGINE_REPO}/..
-gclient sync
-./flutter/tools/gn --unoptimized --runtime-mode debug --simulator --ios
-ninja -C out/ios_debug_sim_unopt
-cd ${ENGINE_REPO}/testing/scenario_app
-./run_ios_tests.sh
+./build_and_run_ios_tests.sh
 ```
 
-## Adding a New Scenario
-
-Create a new subclass of [Scenario](https://github.com/flutter/engine/blob/5d9509ae056b04c30295df27f201f31af9777842/testing/scenario_app/lib/src/scenario.dart#L9) and add it to the
-map in [main.dart](https://github.com/flutter/engine/blob/5d9509ae056b04c30295df27f201f31af9777842/testing/scenario_app/lib/main.dart#L17). For an example, see [animated_color_square.dart](https://github.com/flutter/engine/blob/5d9509ae056b04c30295df27f201f31af9777842/testing/scenario_app/lib/src/animated_color_square.dart#L15), which draws
-a continuously animating colored square that bounces off the sides of the
-viewport.
-
 ### iOS Platform View Tests
 
 For PlatformView tests on iOS, you'll also have to edit the dictionaries in
@@ -41,40 +40,12 @@ Inside that directory you'll find
 `./Build/Products/Debug-iphonesimulator/ScenariosUITests-Runner.app/PlugIns/ScenariosUITests.xctest/` which is where all the images that were
 compared against golden reside.
 
-## Building for iOS
-
-In this folder, after building the `ios_host` and `ios_profile` engine targets,
-run:
-
-```bash
-./compile_ios_aot.sh ../../../out/host_profile ../../../out/ios_profile/clang_x64/
-```
-
-This will create an `App.framework` copy it as well as the correct
-`Flutter.framework` to where the Xcode project expects to find them.
-
-You can then use `xcodebuild` to build the `ios/Scenarios/Scenarios.xcodeproj`,
-or open that in Xcode and build it that way.
-
-Compiling to JIT mode is similar, using the `compile_ios_jit.sh` script.
-
-## Building for Android
-
-In this folder, after building the `host_profile` and `android_profile_arm64`
-engine targets, run:
+## Running for Android
 
 ```bash
-./compile_android_aot.sh ../../../out/host_profile ../../../out/android_profile_arm64/clang_x64/
+./build_and_run_android_tests.sh
 ```
 
-This will produce a suitable `libapp.so` for building with an Android app and
-copy it (along with flutter.jar) to where Gradle will expect to find it to build
-the app in the `android/` folder. The app can be run by opening it in Android
-Studio and running it, or by running `./gradlew assemble` in the `android/`
-folder and installing the APK from the correct folder in
-`android/app/build/outputs/apk`.
-
-
 ### Generating Golden Images on Android
 
 In the `android` directory, run:

testing/scenario_app/build_and_run_android_tests.sh

+#!/bin/sh
+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Runs the Android scenario tests on a connected device.
+# TODO(https://github.com/flutter/flutter/issues/55326): use Flutter tool and
+# just build debug JIT for emulator.
+
+set -e
+
+FLUTTER_ENGINE=android_profile_unopt_arm64
+
+if [ $# -eq 1 ]; then
+  FLUTTER_ENGINE=$1
+fi
+
+cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd
+
+pushd ../../..
+
+if [ ! -d "out/$FLUTTER_ENGINE" ]; then
+  echo "You must GN to generate out/$FLUTTER_ENGINE and its host engine."
+  echo "Example: "
+  echo "  ./tools/gn --android --unoptimized --android-cpu arm64 --runtime-mode profile"
+  echo "  ./tools/gn --unoptimized --runtime-mode profile"
+  echo "to create out/android_profile_unopt_arm64 and out/host_profile_unopt."
+  exit 1
+fi
+
+autoninja -C out/$FLUTTER_ENGINE
+
+popd
+
+./compile_android_aot.sh ../../../out/host_profile_unopt ../../../out/$FLUTTER_ENGINE/clang_x64
+
+./run_android_tests.sh $FLUTTER_ENGINE

testing/scenario_app/build_and_run_ios_tests.sh

+#!/bin/sh
+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+set -e
+
+FLUTTER_ENGINE=ios_debug_sim_unopt
+
+if [ $# -eq 1 ]; then
+  FLUTTER_ENGINE=$1
+fi
+
+cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd
+
+pushd ../../..
+
+if [ ! -d "out/$FLUTTER_ENGINE" ]; then
+  echo "You must GN to generate out/$FLUTTER_ENGINE"
+  echo "Example: "
+  echo "  ./flutter/tools/gn --ios --simulator --unoptimized"
+  echo "  ./flutter/tools/gn --unoptimized"
+  echo "to create out/ios_debug_sim_unopt and out/host_debug_unopt."
+  exit 1
+fi
+
+autoninja -C out/$FLUTTER_ENGINE
+
+popd
+
+./compile_ios_jit.sh ../../../out/host_debug_unopt ../../../out/$FLUTTER_ENGINE/clang_x64
+
+./run_ios_tests.sh $FLUTTER_ENGINE

testing/scenario_app/ios/Scenarios/Scenarios.xcodeproj/project.pbxproj

@@ -308,7 +308,6 @@
 			isa = PBXNativeTarget;
 			buildConfigurationList = 248D76F322E388380012F0C1 /* Build configuration list for PBXNativeTarget "Scenarios" */;
 			buildPhases = (
-				0D8470A8241035AD0030B565 /* Compile iOS JIT */,
 				248D76C322E388370012F0C1 /* Sources */,
 				248D76C422E388370012F0C1 /* Frameworks */,
 				248D76C522E388370012F0C1 /* Resources */,
@@ -451,27 +450,6 @@
 		};
 /* End PBXResourcesBuildPhase section */
 
-/* Begin PBXShellScriptBuildPhase section */
-		0D8470A8241035AD0030B565 /* Compile iOS JIT */ = {
-			isa = PBXShellScriptBuildPhase;
-			buildActionMask = 2147483647;
-			files = (
-			);
-			inputFileListPaths = (
-			);
-			inputPaths = (
-			);
-			name = "Compile iOS JIT";
-			outputFileListPaths = (
-			);
-			outputPaths = (
-			);
-			runOnlyForDeploymentPostprocessing = 0;
-			shellPath = /bin/sh;
-			shellScript = "set -e\ncd ../..\nif [ -z ${FLUTTER_ENGINE+x} ]; then \n  FLUTTER_ENGINE=ios_debug_sim_unopt\nfi\n./compile_ios_jit.sh ../../../out/host_debug_unopt ../../../out/$FLUTTER_ENGINE/clang_x64\n";
-		};
-/* End PBXShellScriptBuildPhase section */
-
 /* Begin PBXSourcesBuildPhase section */
 		248D76C322E388370012F0C1 /* Sources */ = {
 			isa = PBXSourcesBuildPhase;

testing/scenario_app/run_android_tests.sh

@@ -15,8 +15,6 @@ fi
 
 cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd
 
-./compile_android_aot.sh ../../../out/host_profile_unopt_arm64 ../../../out/$FLUTTER_ENGINE/clang_x64
-
 pushd android
 
 set -o pipefail && ./gradlew app:verifyDebugAndroidTestScreenshotTest

testing/scenario_app/run_ios_tests.sh

@@ -15,11 +15,14 @@ fi
 
 cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd
 
+# Delete after LUCI push.
+./compile_ios_jit.sh ../../../out/host_debug_unopt ../../../out/$FLUTTER_ENGINE/clang_x64
+
 pushd ios/Scenarios
 
 set -o pipefail && xcodebuild -sdk iphonesimulator \
   -scheme Scenarios \
-  -destination 'platform=iOS Simulator,OS=13.0,name=iPhone 8' \
+  -destination 'platform=iOS Simulator,name=iPhone 8' \
   test \
   FLUTTER_ENGINE=$FLUTTER_ENGINE | $PRETTY