Firebase test for Platform Views on iOS (#11350)

testing/scenario_app/firebase_xctest.sh

+#!/bin/bash
+
+set -e
+
+"${BASH_SOURCE%/*}/compile_ios_aot.sh" $1 $2
+
+GIT_REVISION=${3:-$(git rev-parse HEAD)}
+BUILD_ID=${4:-$CIRRUS_BUILD_ID}
+
+pushd "${BASH_SOURCE%/*}/ios/Scenarios"
+xcodebuild -project Scenarios.xcodeproj -scheme Scenarios -configuration Debug \
+    -sdk iphoneos \
+    -derivedDataPath DerivedData/Scenarios \
+    build-for-testing
+
+pushd DerivedData/Scenarios/Build/Products
+
+zip -r scenarios.zip Debug-iphoneos Scenarios*.xctestrun
+
+gcloud firebase test ios run --test ./scenarios.zip \
+    --device model=iphone8plus,version=12.0,locale=en_US,orientation=portrait \
+    --xcode-version=10.2 \
+    --results-bucket=gs://flutter_firebase_testlab \
+    --results-dir=engine_scenario_test/$GIT_REVISION/$BUILD_ID \
+
+popd
+popd

testing/scenario_app/ios/Scenarios/.gitignore

+DerivedData/
+

testing/scenario_app/ios/Scenarios/Scenarios.xcodeproj/project.pbxproj

@@ -16,6 +16,7 @@
 		0DB7820222EA493B00E9B371 /* FlutterViewControllerTest.m in Sources */ = {isa = PBXBuildFile; fileRef = 0DB781FC22EA2C0300E9B371 /* FlutterViewControllerTest.m */; };
 		242F37A222E636DE001E83D4 /* Flutter.framework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 246B4E4522E3B61000073EBF /* Flutter.framework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; };
 		242F37A322E636DE001E83D4 /* App.framework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 246B4E4122E3B5F700073EBF /* App.framework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; };
+		244EA6D0230DBE8900B2D26E /* golden_platform_view_D21AP.png in Resources */ = {isa = PBXBuildFile; fileRef = 244EA6CF230DBE8900B2D26E /* golden_platform_view_D21AP.png */; };
 		246B4E4222E3B5F700073EBF /* App.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 246B4E4122E3B5F700073EBF /* App.framework */; };
 		246B4E4622E3B61000073EBF /* Flutter.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 246B4E4522E3B61000073EBF /* Flutter.framework */; };
 		248D76CC22E388370012F0C1 /* AppDelegate.m in Sources */ = {isa = PBXBuildFile; fileRef = 248D76CB22E388370012F0C1 /* AppDelegate.m */; };
@@ -23,7 +24,7 @@
 		248D76DA22E388380012F0C1 /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = 248D76D922E388380012F0C1 /* main.m */; };
 		248D76EF22E388380012F0C1 /* PlatformViewUITests.m in Sources */ = {isa = PBXBuildFile; fileRef = 248D76EE22E388380012F0C1 /* PlatformViewUITests.m */; };
 		248FDFC422FE7CD0009CC7CD /* FlutterEngineTest.m in Sources */ = {isa = PBXBuildFile; fileRef = 248FDFC322FE7CD0009CC7CD /* FlutterEngineTest.m */; };
-		24D47D1B230C79840069DD5E /* golden_platform_view_D21AAP.png in Resources */ = {isa = PBXBuildFile; fileRef = 24D47D1A230C79840069DD5E /* golden_platform_view_D21AAP.png */; };
+		24D47D1B230C79840069DD5E /* golden_platform_view_D211AP.png in Resources */ = {isa = PBXBuildFile; fileRef = 24D47D1A230C79840069DD5E /* golden_platform_view_D211AP.png */; };
 		24D47D1D230CA2700069DD5E /* golden_platform_view_iPhone SE_simulator.png in Resources */ = {isa = PBXBuildFile; fileRef = 24D47D1C230CA2700069DD5E /* golden_platform_view_iPhone SE_simulator.png */; };
 		24F1FB89230B4579005ACE7C /* TextPlatformView.m in Sources */ = {isa = PBXBuildFile; fileRef = 24F1FB87230B4579005ACE7C /* TextPlatformView.m */; };
 /* End PBXBuildFile section */
@@ -84,6 +85,7 @@
 
 /* Begin PBXFileReference section */
 		0DB781FC22EA2C0300E9B371 /* FlutterViewControllerTest.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FlutterViewControllerTest.m; sourceTree = "<group>"; };
+		244EA6CF230DBE8900B2D26E /* golden_platform_view_D21AP.png */ = {isa = PBXFileReference; lastKnownFileType = image.png; path = golden_platform_view_D21AP.png; sourceTree = "<group>"; };
 		246B4E4122E3B5F700073EBF /* App.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; path = App.framework; sourceTree = "<group>"; };
 		246B4E4522E3B61000073EBF /* Flutter.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; path = Flutter.framework; sourceTree = "<group>"; };
 		248D76C722E388370012F0C1 /* Scenarios.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = Scenarios.app; sourceTree = BUILT_PRODUCTS_DIR; };
@@ -98,7 +100,7 @@
 		248D76EE22E388380012F0C1 /* PlatformViewUITests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = PlatformViewUITests.m; sourceTree = "<group>"; };
 		248D76F022E388380012F0C1 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
 		248FDFC322FE7CD0009CC7CD /* FlutterEngineTest.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FlutterEngineTest.m; sourceTree = "<group>"; };
-		24D47D1A230C79840069DD5E /* golden_platform_view_D21AAP.png */ = {isa = PBXFileReference; lastKnownFileType = image.png; path = golden_platform_view_D21AAP.png; sourceTree = "<group>"; };
+		24D47D1A230C79840069DD5E /* golden_platform_view_D211AP.png */ = {isa = PBXFileReference; lastKnownFileType = image.png; path = golden_platform_view_D211AP.png; sourceTree = "<group>"; };
 		24D47D1C230CA2700069DD5E /* golden_platform_view_iPhone SE_simulator.png */ = {isa = PBXFileReference; lastKnownFileType = image.png; path = "golden_platform_view_iPhone SE_simulator.png"; sourceTree = "<group>"; };
 		24D47D1E230CA4480069DD5E /* README.md */ = {isa = PBXFileReference; lastKnownFileType = net.daringfireball.markdown; path = README.md; sourceTree = "<group>"; };
 		24F1FB87230B4579005ACE7C /* TextPlatformView.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = TextPlatformView.m; sourceTree = "<group>"; };
@@ -182,8 +184,9 @@
 		248D76ED22E388380012F0C1 /* ScenariosUITests */ = {
 			isa = PBXGroup;
 			children = (
+				244EA6CF230DBE8900B2D26E /* golden_platform_view_D21AP.png */,
 				24D47D1C230CA2700069DD5E /* golden_platform_view_iPhone SE_simulator.png */,
-				24D47D1A230C79840069DD5E /* golden_platform_view_D21AAP.png */,
+				24D47D1A230C79840069DD5E /* golden_platform_view_D211AP.png */,
 				248D76EE22E388380012F0C1 /* PlatformViewUITests.m */,
 				248D76F022E388380012F0C1 /* Info.plist */,
 				24D47D1E230CA4480069DD5E /* README.md */,
@@ -321,8 +324,9 @@
 			isa = PBXResourcesBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
-				24D47D1B230C79840069DD5E /* golden_platform_view_D21AAP.png in Resources */,
+				24D47D1B230C79840069DD5E /* golden_platform_view_D211AP.png in Resources */,
 				24D47D1D230CA2700069DD5E /* golden_platform_view_iPhone SE_simulator.png in Resources */,
+				244EA6D0230DBE8900B2D26E /* golden_platform_view_D21AP.png in Resources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -424,7 +428,7 @@
 				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
 				GCC_WARN_UNUSED_FUNCTION = YES;
 				GCC_WARN_UNUSED_VARIABLE = YES;
-				IPHONEOS_DEPLOYMENT_TARGET = 12.2;
+				IPHONEOS_DEPLOYMENT_TARGET = 12.0;
 				MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
 				MTL_FAST_MATH = YES;
 				ONLY_ACTIVE_ARCH = YES;
@@ -478,7 +482,7 @@
 				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
 				GCC_WARN_UNUSED_FUNCTION = YES;
 				GCC_WARN_UNUSED_VARIABLE = YES;
-				IPHONEOS_DEPLOYMENT_TARGET = 12.2;
+				IPHONEOS_DEPLOYMENT_TARGET = 12.0;
 				MTL_ENABLE_DEBUG_INFO = NO;
 				MTL_FAST_MATH = YES;
 				SDKROOT = iphoneos;

testing/scenario_app/ios/Scenarios/Scenarios.xcodeproj/project.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings

+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict/>
+</plist>

testing/scenario_app/ios/Scenarios/Scenarios.xcodeproj/xcshareddata/xcschemes/Scenarios.xcscheme

+<?xml version="1.0" encoding="UTF-8"?>
+<Scheme
+   LastUpgradeVersion = "1020"
+   version = "1.3">
+   <BuildAction
+      parallelizeBuildables = "YES"
+      buildImplicitDependencies = "YES">
+      <BuildActionEntries>
+         <BuildActionEntry
+            buildForTesting = "YES"
+            buildForRunning = "YES"
+            buildForProfiling = "YES"
+            buildForArchiving = "YES"
+            buildForAnalyzing = "YES">
+            <BuildableReference
+               BuildableIdentifier = "primary"
+               BlueprintIdentifier = "248D76C622E388370012F0C1"
+               BuildableName = "Scenarios.app"
+               BlueprintName = "Scenarios"
+               ReferencedContainer = "container:Scenarios.xcodeproj">
+            </BuildableReference>
+         </BuildActionEntry>
+      </BuildActionEntries>
+   </BuildAction>
+   <TestAction
+      buildConfiguration = "Debug"
+      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      shouldUseLaunchSchemeArgsEnv = "YES">
+      <Testables>
+         <TestableReference
+            skipped = "NO"
+            testExecutionOrdering = "random">
+            <BuildableReference
+               BuildableIdentifier = "primary"
+               BlueprintIdentifier = "248D76DE22E388380012F0C1"
+               BuildableName = "ScenariosTests.xctest"
+               BlueprintName = "ScenariosTests"
+               ReferencedContainer = "container:Scenarios.xcodeproj">
+            </BuildableReference>
+         </TestableReference>
+         <TestableReference
+            skipped = "NO"
+            testExecutionOrdering = "random">
+            <BuildableReference
+               BuildableIdentifier = "primary"
+               BlueprintIdentifier = "248D76E922E388380012F0C1"
+               BuildableName = "ScenariosUITests.xctest"
+               BlueprintName = "ScenariosUITests"
+               ReferencedContainer = "container:Scenarios.xcodeproj">
+            </BuildableReference>
+         </TestableReference>
+      </Testables>
+      <MacroExpansion>
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "248D76C622E388370012F0C1"
+            BuildableName = "Scenarios.app"
+            BlueprintName = "Scenarios"
+            ReferencedContainer = "container:Scenarios.xcodeproj">
+         </BuildableReference>
+      </MacroExpansion>
+      <AdditionalOptions>
+      </AdditionalOptions>
+   </TestAction>
+   <LaunchAction
+      buildConfiguration = "Debug"
+      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      launchStyle = "0"
+      useCustomWorkingDirectory = "NO"
+      ignoresPersistentStateOnLaunch = "NO"
+      debugDocumentVersioning = "YES"
+      debugServiceExtension = "internal"
+      allowLocationSimulation = "YES">
+      <BuildableProductRunnable
+         runnableDebuggingMode = "0">
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "248D76C622E388370012F0C1"
+            BuildableName = "Scenarios.app"
+            BlueprintName = "Scenarios"
+            ReferencedContainer = "container:Scenarios.xcodeproj">
+         </BuildableReference>
+      </BuildableProductRunnable>
+      <AdditionalOptions>
+      </AdditionalOptions>
+   </LaunchAction>
+   <ProfileAction
+      buildConfiguration = "Release"
+      shouldUseLaunchSchemeArgsEnv = "YES"
+      savedToolIdentifier = ""
+      useCustomWorkingDirectory = "NO"
+      debugDocumentVersioning = "YES">
+      <BuildableProductRunnable
+         runnableDebuggingMode = "0">
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "248D76C622E388370012F0C1"
+            BuildableName = "Scenarios.app"
+            BlueprintName = "Scenarios"
+            ReferencedContainer = "container:Scenarios.xcodeproj">
+         </BuildableReference>
+      </BuildableProductRunnable>
+   </ProfileAction>
+   <AnalyzeAction
+      buildConfiguration = "Debug">
+   </AnalyzeAction>
+   <ArchiveAction
+      buildConfiguration = "Release"
+      revealArchiveInOrganizer = "YES">
+   </ArchiveAction>
+</Scheme>

testing/scenario_app/ios/Scenarios/Scenarios/Info.plist

@@ -24,14 +24,14 @@
 	<array>
 		<string>arm64</string>
 	</array>
+	<key>UIRequiresFullScreen</key>
+	<true/>
 	<key>UISupportedInterfaceOrientations</key>
 	<array>
 		<string>UIInterfaceOrientationPortrait</string>
 		<string>UIInterfaceOrientationLandscapeLeft</string>
 		<string>UIInterfaceOrientationLandscapeRight</string>
 	</array>
-	<key>io.flutter.embedded_views_preview</key>
-	<true/>
 	<key>UISupportedInterfaceOrientations~ipad</key>
 	<array>
 		<string>UIInterfaceOrientationPortrait</string>
@@ -39,5 +39,7 @@
 		<string>UIInterfaceOrientationLandscapeLeft</string>
 		<string>UIInterfaceOrientationLandscapeRight</string>
 	</array>
+	<key>io.flutter.embedded_views_preview</key>
+	<true/>
 </dict>
 </plist>

testing/scenario_app/ios/Scenarios/ScenariosUITests/PlatformViewUITests.m

@@ -31,15 +31,20 @@
   UIImage* golden = [[UIImage alloc] initWithContentsOfFile:path];
 
   XCUIScreenshot* screenshot = [[XCUIScreen mainScreen] screenshot];
+  XCTAttachment* attachment = [XCTAttachment attachmentWithScreenshot:screenshot];
+  attachment.lifetime = XCTAttachmentLifetimeKeepAlways;
+  [self addAttachment:attachment];
+
   if (golden) {
     XCTAttachment* goldenAttachment = [XCTAttachment attachmentWithImage:golden];
     goldenAttachment.lifetime = XCTAttachmentLifetimeKeepAlways;
     [self addAttachment:goldenAttachment];
+  } else {
+    XCTFail(@"This test will fail - no golden named %@ found. Follow the steps in the "
+            @"README to add a new golden.",
+            goldenName);
   }
 
-  XCTAttachment* attachment = [XCTAttachment attachmentWithScreenshot:screenshot];
-  attachment.lifetime = XCTAttachmentLifetimeKeepAlways;
-  [self addAttachment:attachment];
   XCTAssertTrue([self compareImage:golden toOther:screenshot.image]);
 }
 

testing/scenario_app/ios/Scenarios/ScenariosUITests/README.md

@@ -11,3 +11,9 @@ names are taken from the environment.
 New devices require running the test on the device, gathering the attachment
 and verifying it manually, and then adding an appropriately named file to
 this folder.
+
+If the test is attempted on a new device, the log will contain a message
+indicating the file name it expected to find. The test will continue and fail,
+but will contain an attachment with the expected screen shot. If the screen
+shot looks good, add it with the correct name to the project and run the test
+again - it should pass this time.