chore: Update unstable tests (#1410)

.azure-templates/bootstrap_steps.yml

+steps:
+  - task: NodeTool@0
+    inputs:
+      versionSpec: "$(NODE_VERSION)"
+  - script: |
+      npm config delete prefix
+      npm config set prefix $NVM_DIR/versions/node/`node --version`
+      node --version
+
+      npm install -g appium@beta

azure-pipelines.yml

@@ -9,52 +9,55 @@ pool:
 variables:
   ANDROID_EMU_NAME: test
   ANDROID_EMU_ABI: x86
-  ANDROID_EMU_TARGET: android-27
-  ANDROID_EMU_TAG: google_apis
+  ANDROID_EMU_TARGET: android-28
+  ANDROID_EMU_TAG: default
   XCODE_VERSION: 11.5
   IOS_PLATFORM_VERSION: 13.5
   IOS_DEVICE_NAME: iPhone X
+  NODE_VERSION: 12.x
+  JDK_VERSION: 1.8
 
 jobs:
-- job: E2E_Tests
-  timeoutInMinutes: '60'
+- job: Android_E2E_Tests
+#  timeoutInMinutes: '90'
   steps:
-  - task: NodeTool@0
-    inputs:
-      versionSpec: '12.x'
-
+  - template: .azure-templates/bootstrap_steps.yml
   - script: |
-      echo "Configuring Environment"
       echo "y" | $ANDROID_HOME/tools/bin/sdkmanager --install 'system-images;$(ANDROID_EMU_TARGET);$(ANDROID_EMU_TAG);$(ANDROID_EMU_ABI)'
       echo "no" | $ANDROID_HOME/tools/bin/avdmanager create avd -n "$(ANDROID_EMU_NAME)" -k 'system-images;$(ANDROID_EMU_TARGET);$(ANDROID_EMU_TAG);$(ANDROID_EMU_ABI)' --force
       echo $ANDROID_HOME/emulator/emulator -list-avds
 
       echo "Starting emulator"
-      nohup $ANDROID_HOME/emulator/emulator -avd "$(ANDROID_EMU_NAME)" -no-snapshot > /dev/null 2>&1 &
+      nohup $ANDROID_HOME/emulator/emulator -avd "$(ANDROID_EMU_NAME)" -no-snapshot -delay-adb > /dev/null 2>&1 &
       $ANDROID_HOME/platform-tools/adb wait-for-device
-      while [[ $? -ne 0 ]]; do sleep 1; $ANDROID_HOME/platform-tools/adb shell pm list packages; done;
-      $ANDROID_HOME/platform-tools/adb devices
+      $ANDROID_HOME/platform-tools/adb devices -l
       echo "Emulator started"
-
+    displayName: Emulator configuration
+  - task: Gradle@2
+    inputs:
+      gradleWrapperFile: 'gradlew'
+      gradleOptions: '-Xmx3072m'
+      javaHomeOption: 'JDKVersion'
+      jdkVersionOption: "$(JDK_VERSION)"
+      jdkArchitectureOption: 'x64'
+      publishJUnitResults: true
+      tasks: 'build'
+      options: 'uiAutomationTest -x checkstyleTest -x test -x signMavenJavaPublication'
+- job: iOS_E2E_Tests
+#  timeoutInMinutes: '90'
+  steps:
+  - template: .azure-templates/bootstrap_steps.yml
+  - script: |
       sudo xcode-select -s /Applications/Xcode_$(XCODE_VERSION).app/Contents/Developer
       xcrun simctl list
-
-      npm config delete prefix
-      npm config set prefix $NVM_DIR/versions/node/`node --version`
-      node --version
-
-      npm install -g appium@beta
-      appium --version
-
-      java -version
-
+    displayName: Simulator configuration
   - task: Gradle@2
     inputs:
       gradleWrapperFile: 'gradlew'
       gradleOptions: '-Xmx3072m'
       javaHomeOption: 'JDKVersion'
-      jdkVersionOption: '1.8'
+      jdkVersionOption: "$(JDK_VERSION)"
       jdkArchitectureOption: 'x64'
       publishJUnitResults: true
       tasks: 'build'
-      options: 'xcuiTest uiAutomationTest -x checkstyleTest -x test -x signMavenJavaPublication'
+      options: 'xcuiTest -x checkstyleTest -x test -x signMavenJavaPublication'

build.gradle

@@ -227,7 +227,7 @@ task uiAutomationTest( type: Test ) {
     testLogging.showStandardStreams = true
     testLogging.exceptionFormat = 'full'
     filter {
-        includeTestsMatching '*.SettingTest'
+        includeTestsMatching 'io.appium.java_client.android.SettingTest'
         includeTestsMatching 'io.appium.java_client.android.ClipboardTest'
         includeTestsMatching '*.AndroidAppStringsTest'
     }

src/test/java/io/appium/java_client/TestUtils.java

 package io.appium.java_client;
 
+import org.openqa.selenium.TimeoutException;
+
 import java.io.IOException;
 import java.net.DatagramSocket;
 import java.net.InetAddress;
@@ -9,6 +11,8 @@ import java.net.UnknownHostException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.time.Duration;
+import java.util.function.Supplier;
 
 public class TestUtils {
     public static String getLocalIp4Address() throws SocketException, UnknownHostException {
@@ -34,4 +38,30 @@ public class TestUtils {
             throw new RuntimeException(e);
         }
     }
+
+    public static void waitUntilTrue(Supplier<Boolean> func, Duration timeout, Duration interval) {
+        long started = System.currentTimeMillis();
+        RuntimeException lastError = null;
+        while (System.currentTimeMillis() - started < timeout.toMillis()) {
+            lastError = null;
+            try {
+                Boolean result = func.get();
+                if (result != null && result) {
+                    return;
+                }
+                //noinspection BusyWait
+                Thread.sleep(interval.toMillis());
+            } catch (RuntimeException | InterruptedException e) {
+                if (e instanceof InterruptedException) {
+                    throw new RuntimeException(e);
+                } else {
+                    lastError = (RuntimeException) e;
+                }
+            }
+        }
+        if (lastError != null) {
+            throw lastError;
+        }
+        throw new TimeoutException(String.format("Condition unmet after %sms timeout", timeout.toMillis()));
+    }
 }

src/test/java/io/appium/java_client/android/BaseAndroidTest.java

@@ -39,7 +39,6 @@ public class BaseAndroidTest {
     @BeforeClass public static void beforeClass() {
         service = AppiumDriverLocalService.buildDefaultService();
         service.start();
-
         if (service == null || !service.isRunning()) {
             throw new AppiumServerHasNotBeenStartedLocallyException(
                 "An appium server node is not started!");

src/test/java/io/appium/java_client/ios/IOSDriverTest.java

@@ -16,22 +16,18 @@
 
 package io.appium.java_client.ios;
 
-import static org.hamcrest.Matchers.empty;
+import static io.appium.java_client.TestUtils.waitUntilTrue;
+import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.lessThan;
-import static org.hamcrest.Matchers.not;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import io.appium.java_client.MobileElement;
 import io.appium.java_client.appmanagement.ApplicationState;
 import io.appium.java_client.remote.HideKeyboardStrategy;
-import io.appium.java_client.remote.MobileCapabilityType;
 import org.junit.Ignore;
 import org.junit.Test;
 
@@ -90,8 +86,8 @@ public class IOSDriverTest extends AppIOSTest {
     }
 
     @Test public void pullFileTest() {
-        byte[] data = driver.pullFile("@io.appium.TestApp/TestApp");
-        assert (data.length > 0);
+        byte[] data = driver.pullFile(String.format("@%s/TestApp", BUNDLE_ID));
+        assertThat(data.length, greaterThan(0));
     }
 
     @Test public void keyboardTest() {
@@ -106,31 +102,25 @@ public class IOSDriverTest extends AppIOSTest {
         assertThat(System.currentTimeMillis() - msStarted, greaterThan(3000L));
     }
 
-    @Test public void applicationsManagementTest() throws InterruptedException {
-        // This only works since Xcode9
-        try {
-            if (Double.parseDouble(
-                    (String) driver.getCapabilities()
-                            .getCapability(MobileCapabilityType.PLATFORM_VERSION)) < 11) {
-                return;
-            }
-        } catch (NumberFormatException | NullPointerException e) {
-            return;
-        }
+    @Test public void applicationsManagementTest() {
         assertThat(driver.queryAppState(BUNDLE_ID), equalTo(ApplicationState.RUNNING_IN_FOREGROUND));
-        Thread.sleep(500);
         driver.runAppInBackground(Duration.ofSeconds(-1));
-        assertThat(driver.queryAppState(BUNDLE_ID), lessThan(ApplicationState.RUNNING_IN_FOREGROUND));
-        Thread.sleep(500);
+        waitUntilTrue(
+                () -> driver.queryAppState(BUNDLE_ID).ordinal() < ApplicationState.RUNNING_IN_FOREGROUND.ordinal(),
+                Duration.ofSeconds(10), Duration.ofSeconds(1));
         driver.activateApp(BUNDLE_ID);
-        assertThat(driver.queryAppState(BUNDLE_ID), equalTo(ApplicationState.RUNNING_IN_FOREGROUND));
+        waitUntilTrue(
+                () -> driver.queryAppState(BUNDLE_ID) == ApplicationState.RUNNING_IN_FOREGROUND,
+                Duration.ofSeconds(10), Duration.ofSeconds(1));
     }
 
     @Test public void putAIntoBackgroundWithoutRestoreTest() {
-        assertThat(driver.findElementsById("IntegerA"), is(not(empty())));
+        waitUntilTrue(() -> !driver.findElementsById("IntegerA").isEmpty(),
+                Duration.ofSeconds(10), Duration.ofSeconds(1));
         driver.runAppInBackground(Duration.ofSeconds(-1));
-        assertThat(driver.findElementsById("IntegerA"), is(empty()));
-        driver.launchApp();
+        waitUntilTrue(() -> driver.findElementsById("IntegerA").isEmpty(),
+                Duration.ofSeconds(10), Duration.ofSeconds(1));
+        driver.activateApp(BUNDLE_ID);
     }
 
     @Ignore
@@ -138,6 +128,7 @@ public class IOSDriverTest extends AppIOSTest {
         driver.toggleTouchIDEnrollment(true);
         driver.performTouchID(true);
         driver.performTouchID(false);
+        //noinspection SimplifiableAssertion
         assertEquals(true, true);
     }
 }