Use custom mobile testing API (#53871)

* Use custom Android testing API

* Fix for non-mobile cases

* Use Collection Fixtures for mobile only

* Update src/tests/Common/Coreclr.TestWrapper/MobileAppHandler.cs
Co-authored-by: NPřemek Vysoký <premek.vysoky@microsoft.com>

* Update src/tests/Common/Coreclr.TestWrapper/MobileAppHandler.cs
Co-authored-by: NPřemek Vysoký <premek.vysoky@microsoft.com>

* Update src/tests/run.proj
Co-authored-by: NPřemek Vysoký <premek.vysoky@microsoft.com>

* Fix the ios app extension

* Add back Path and remove redundant namespace string

* Add missing class name Path
Co-authored-by: NPřemek Vysoký <premek.vysoky@microsoft.com>

src/tests/Common/CLRTest.Execute.Bash.targets

@@ -363,7 +363,7 @@ else
 	HARNESS_RUNNER="xharness"
 fi
 
-$__Command $HARNESS_RUNNER android test --instrumentation="net.dot.MonoRunner" --package-name="net.dot.$__Category" --app="$__TestBinaryBase/$__Category.apk" --output-directory="$__OutputDir" --arg=entrypoint:libname=$(MsBuildProjectName).dll --expected-exit-code=100 -v
+$__Command $HARNESS_RUNNER android run --instrumentation="net.dot.MonoRunner" --package-name="net.dot.$__Category" --app="$__TestBinaryBase/$__Category.apk" --output-directory="$__OutputDir" --arg=entrypoint:libname=$(MsBuildProjectName).dll --expected-exit-code=100 -v
 CLRTestExitCode=$?
 
 # Exist code of xharness is zero when tests finished successfully

src/tests/Common/CLRTest.Execute.Batch.targets

@@ -329,7 +329,7 @@ IF NOT "%XHARNESS_CLI_PATH%"=="" (
   set HARNESS_RUNNER=xharness
 )
 
-%__Command% %HARNESS_RUNNER% android test --instrumentation="net.dot.MonoRunner" --package-name="net.dot.%__Category%" --app="%__TestBinaryBase%\%__Category%.apk" --output-directory="%__OutputDir%" --arg=entrypoint:libname=$(MsBuildProjectName).dll --expected-exit-code=100 -v
+%__Command% %HARNESS_RUNNER% android run --instrumentation="net.dot.MonoRunner" --package-name="net.dot.%__Category%" --app="%__TestBinaryBase%\%__Category%.apk" --output-directory="%__OutputDir%" --arg=entrypoint:libname=$(MsBuildProjectName).dll --expected-exit-code=100 -v
 set CLRTestExitCode=!ERRORLEVEL!
 set CLRTestExpectedExitCode=0
     ]]></BatchCLRTestLaunchCmds>

src/tests/Common/Coreclr.TestWrapper/Coreclr.TestWrapper.csproj

@@ -9,6 +9,7 @@
 
   <ItemGroup>
     <Compile Include="CoreclrTestWrapperLib.cs" />
+    <Compile Include="MobileAppHandler.cs" />
   </ItemGroup>
 
 </Project>

src/tests/Common/Coreclr.TestWrapper/MobileAppHandler.cs

+using System;
+using System.IO;
+using System.Diagnostics;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace CoreclrTestLib
+{
+    public class MobileAppHandler
+    {
+        public void InstallMobileApp(string platform, string category, string testBinaryBase, string reportBase)
+        {
+            HandleMobileApp("install", platform, category, testBinaryBase, reportBase);
+        }
+
+        public void UninstallMobileApp(string platform, string category, string testBinaryBase, string reportBase)
+        {
+            HandleMobileApp("uninstall", platform, category, testBinaryBase, reportBase);
+        }
+
+        private static void HandleMobileApp(string action, string platform, string category, string testBinaryBase, string reportBase)
+        {
+            //install or uninstall mobile app
+            string outputFile = Path.Combine(reportBase, action, $"{category}_{action}.output.txt");
+            string errorFile = Path.Combine(reportBase, action, $"{category}_{action}.error.txt");
+            string dotnetCmd_raw = System.Environment.GetEnvironmentVariable("__TestDotNetCmd");
+            string dotnetCmd;
+            string xharnessCmd_raw = System.Environment.GetEnvironmentVariable("XHARNESS_CLI_PATH");
+            string xharnessCmd;
+            string cmdStr;
+            string appExtension;
+            int timeout = 240000; // Set timeout to 4 mins, because the installation on Android arm64/32 devices could take up to 4 mins on CI
+
+            if(String.IsNullOrEmpty(dotnetCmd_raw))
+            {
+                dotnetCmd = "dotnet";
+            }
+            else
+            {
+                dotnetCmd = dotnetCmd_raw;
+            }
+
+            if(String.IsNullOrEmpty(xharnessCmd_raw))
+            {
+                xharnessCmd = "xharness";
+            }
+            else
+            {
+                xharnessCmd = $"exec {xharnessCmd_raw}";
+            }
+
+            if(platform == "android")
+            {
+                appExtension = "apk";
+            }
+            else
+            {
+                appExtension = "app";
+            }
+
+            cmdStr = $"{dotnetCmd} {xharnessCmd} {platform} {action} --package-name=net.dot.{category} --app={testBinaryBase}/{category}.{appExtension} --output-directory={reportBase}/install";
+
+            Directory.CreateDirectory(Path.Combine(reportBase, action));
+            var outputStream = new FileStream(outputFile, FileMode.Create);
+            var errorStream = new FileStream(errorFile, FileMode.Create);
+
+            using (var outputWriter = new StreamWriter(outputStream))
+            using (var errorWriter = new StreamWriter(errorStream))
+            using (Process process = new Process())
+            {
+                if (OperatingSystem.IsWindows())
+                {
+                    process.StartInfo.FileName = "cmd.exe";
+                }
+                else
+                {
+                    process.StartInfo.FileName = "/bin/bash";
+                }
+
+                process.StartInfo.Arguments = ConvertCmd2Arg(cmdStr);
+                process.StartInfo.UseShellExecute = false;
+                process.StartInfo.RedirectStandardOutput = true;
+                process.StartInfo.RedirectStandardError = true;
+
+                DateTime startTime = DateTime.Now;
+                process.Start();
+
+                var cts = new CancellationTokenSource();
+                Task copyOutput = process.StandardOutput.BaseStream.CopyToAsync(outputStream, 4096, cts.Token);
+                Task copyError = process.StandardError.BaseStream.CopyToAsync(errorStream, 4096, cts.Token);
+
+                if (process.WaitForExit(timeout))
+                {
+                    Task.WaitAll(copyOutput, copyError);
+                }
+                else
+                {
+                    //Time out
+                    DateTime endTime = DateTime.Now;
+
+                    try
+                    {
+                        cts.Cancel();
+                    }
+                    catch {}
+
+                    outputWriter.WriteLine("\ncmdLine:{0} Timed Out (timeout in milliseconds: {1}, start: {3}, end: {4})",
+                            cmdStr, timeout, startTime.ToString(), endTime.ToString());
+                    errorWriter.WriteLine("\ncmdLine:{0} Timed Out (timeout in milliseconds: {1}, start: {3}, end: {4})",
+                            cmdStr, timeout, startTime.ToString(), endTime.ToString());
+                    
+                    process.Kill(entireProcessTree: true);
+                }
+
+                outputWriter.Flush();
+                errorWriter.Flush();
+            }
+        }
+
+        private static string ConvertCmd2Arg(string cmd)
+        {
+            cmd.Replace("\"", "\"\"");
+            var result = $"-c \"{cmd}\"";
+            return result;
+        }
+    }
+}

src/tests/run.proj

@@ -181,9 +181,13 @@ $(_XunitEpilog)
       <Category Condition="'$(RunningOnUnix)' == 'true'" >$([System.String]::Copy('$(CategoryWithSlash)').Replace('/','.'))</Category>
       <XunitWrapper>$(Category).XUnitWrapper</XunitWrapper>
       <XunitWrapperSrcDir>$(XunitWrapperGeneratedCSDirBase)$(Category)</XunitWrapperSrcDir>
+      <MobilePlatform Condition=" '$(TargetOS)' == 'Android' ">android</MobilePlatform>
+      <MobilePlatform Condition=" '$(TargetOS)' == 'iOS' Or '$(TargetOS)' == 'iOSSimulator' ">apple</MobilePlatform>
+      <IsMobile>false</IsMobile>
+      <IsMobile Condition=" '$(TargetOS)' == 'Android' Or '$(TargetOS)' == 'iOS' Or '$(TargetOS)' == 'iOSSimulator' ">true</IsMobile>
     </PropertyGroup>
     <PropertyGroup>
-      <_XunitProlog Condition=" '$(_XunitProlog)'=='' ">
+      <_XunitProlog Condition=" '$(_XunitProlog)'=='' and '$(IsMobile)'=='false' ">
         <![CDATA[
 using Xunit%3B
 using Xunit.Abstractions%3B
@@ -208,37 +212,105 @@ namespace $([System.String]::Copy($(Category)).Replace(".","_").Replace("\","").
 
             static _Global()
             {
-                reportBase = System.Environment.GetEnvironmentVariable(%22XunitTestReportDirBase%22)%3B
-                testBinaryBase = System.IO.Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)%3B
-                coreRoot = System.Environment.GetEnvironmentVariable(%22CORE_ROOT%22)%3B
+                reportBase = Environment.GetEnvironmentVariable(%22XunitTestReportDirBase%22)%3B
+                testBinaryBase = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)%3B
+                coreRoot = Environment.GetEnvironmentVariable(%22CORE_ROOT%22)%3B
                 category = "$([System.String]::Copy($(Category)).Replace(".","_").Replace("\","").Replace("-","_"))"%3B
-                helixUploadRoot = System.Environment.GetEnvironmentVariable(%22HELIX_WORKITEM_UPLOAD_ROOT%22)%3B
+                helixUploadRoot = Environment.GetEnvironmentVariable(%22HELIX_WORKITEM_UPLOAD_ROOT%22)%3B
                 if (!String.IsNullOrEmpty(helixUploadRoot)) {
-                    reportBase = System.IO.Path.Combine(System.IO.Path.GetFullPath(helixUploadRoot), "Reports")%3B
+                    reportBase = Path.Combine(Path.GetFullPath(helixUploadRoot), "Reports")%3B
                 }
 
                 if (String.IsNullOrEmpty(reportBase)) {
-                    reportBase = System.IO.Path.Combine(testBinaryBase, "Reports")%3B
+                    reportBase = Path.Combine(testBinaryBase, "Reports")%3B
                 }
                 else
                 {
-                    reportBase = System.IO.Path.GetFullPath(reportBase)%3B
+                    reportBase = Path.GetFullPath(reportBase)%3B
                 }
 
                 if (String.IsNullOrEmpty(coreRoot)) {
                     throw new ArgumentException("Environment variable CORE_ROOT is not set")%3B
                 }
 
-                coreRoot = System.IO.Path.GetFullPath(coreRoot)%3B
+                coreRoot = Path.GetFullPath(coreRoot)%3B
 
-                string operatingSystem = System.Environment.GetEnvironmentVariable("OS")%3B
+                string operatingSystem = Environment.GetEnvironmentVariable("OS")%3B
                 runningInWindows = (operatingSystem != null && operatingSystem.StartsWith("Windows"))%3B
             }
         }
-
 ]]>
       </_XunitProlog>
+      <_XunitProlog Condition=" '$(_XunitProlog)'=='' and '$(IsMobile)'=='true' ">
+        <![CDATA[
+using Xunit%3B
+using Xunit.Abstractions%3B
+using System%3B
+using System.Collections.Generic%3B
+using System.Diagnostics%3B
+using System.Reflection%3B
+using System.Text.RegularExpressions%3B
+using CoreclrTestLib%3B
+using System.IO%3B
 
+namespace $([System.String]::Copy($(Category)).Replace(".","_").Replace("\","").Replace("-","_"))
+{
+        public class _Global : IDisposable
+        {
+            public bool runningInWindows%3B
+            public string reportBase%3B
+            public string testBinaryBase%3B
+            public string coreRoot%3B
+            public string category%3B
+            public string helixUploadRoot%3B
+            public MobileAppHandler handler = new MobileAppHandler()%3B
+
+            public _Global()
+            {
+                reportBase = Environment.GetEnvironmentVariable(%22XunitTestReportDirBase%22)%3B
+                testBinaryBase = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)%3B
+                coreRoot = Environment.GetEnvironmentVariable(%22CORE_ROOT%22)%3B
+                category = "$([System.String]::Copy($(Category)).Replace(".","_").Replace("\","").Replace("-","_"))"%3B
+                helixUploadRoot = Environment.GetEnvironmentVariable(%22HELIX_WORKITEM_UPLOAD_ROOT%22)%3B
+                if (!String.IsNullOrEmpty(helixUploadRoot)) {
+                    reportBase = Path.Combine(Path.GetFullPath(helixUploadRoot), "Reports")%3B
+                }
+
+                if (String.IsNullOrEmpty(reportBase)) {
+                    reportBase = Path.Combine(testBinaryBase, %22Reports%22)%3B
+                }
+                else
+                {
+                    reportBase = Path.GetFullPath(reportBase)%3B
+                }
+
+                if (String.IsNullOrEmpty(coreRoot)) {
+                    throw new ArgumentException("Environment variable CORE_ROOT is not set")%3B
+                }
+
+                coreRoot = Path.GetFullPath(coreRoot)%3B
+
+                string operatingSystem = Environment.GetEnvironmentVariable("OS")%3B
+                runningInWindows = (operatingSystem != null && operatingSystem.StartsWith("Windows"))%3B
+
+                handler.InstallMobileApp(%22$(MobilePlatform)%22, category, testBinaryBase, reportBase)%3B
+            }
+
+            public void Dispose()
+            {
+                handler.UninstallMobileApp(%22$(MobilePlatform)%22, category, testBinaryBase, reportBase)%3B
+            }
+        }
+
+        [CollectionDefinition("Runtime test collection")]
+        public class RuntimeTestCollection : ICollectionFixture<_Global>
+        {
+            // This class has no code, and is never created. Its purpose is simply
+            // to be the place to apply [CollectionDefinition] and all the
+            // ICollectionFixture<> interfaces.
+        }
+]]>
+      </_XunitProlog>
       <_XunitEpilog Condition=" '$(_XunitEpilog)'=='' ">
       <![CDATA[
 }
@@ -271,7 +343,7 @@ namespace $([System.String]::Copy($(Category)).Replace(".","_").Replace("\","").
         <FactName Condition="'$(RunningOnUnix)' == 'true'" >_$([MSBuild]::ValueOrDefault(%(AllCMDs.RelativeToCMDDIR),"").Replace(".","_").Replace("/","_").Replace("-","_"))</FactName>
         <ClassName Condition="'$(RunningOnUnix)' == 'true'" >_$([MSBuild]::ValueOrDefault(%(AllCMDs.RelativeToCMDDIR), '').Replace($(TestScriptExtension),'').Replace('.','_').Replace('/','_').Replace('-','_'))</ClassName>
         <TestGroup>%(AllCMDs.TestGroup)</TestGroup>
-        <XUnitFact >
+        <XUnitFact Condition=" '$(IsMobile)'=='false' ">
           <![CDATA[
 
         public class %(AllCommands.ClassName)
@@ -369,10 +441,117 @@ namespace $([System.String]::Copy($(Category)).Replace(".","_").Replace("\","").
                         output.WriteLine(line)%3B
                     }
 
+                    Assert.True(ret == CoreclrTestWrapperLib.EXIT_SUCCESS_CODE, string.Join(Environment.NewLine, testOutput))%3B
+                }
+            }
+        }
+
+        ]]>
+        </XUnitFact>
+        <XUnitFact Condition=" '$(IsMobile)'=='true' ">
+          <![CDATA[
+        [Collection("Runtime test collection")]
+        public class %(AllCommands.ClassName)
+        {
+            private readonly ITestOutputHelper output%3B
+            _Global globalVar%3B
+
+            public %(AllCommands.ClassName)(ITestOutputHelper output, _Global globalVar)
+            {
+                this.output = output%3B
+                this.globalVar = globalVar%3B
+            }
+
+            [Fact(DisplayName=@"%(AllCommands.DisplayName)")]
+            [Trait("TestGroup", "%(AllCommands.TestGroup)")]
+            public void %(AllCommands.FactName)()
+            {
+                int ret = -100%3B
+                string outputFile = null%3B
+                string errorFile = null%3B
+                string testExecutable = null%3B
+                string outputDir = null%3B
+                Exception infraEx = null%3B
+
+                try
+                {
+                  CoreclrTestWrapperLib wrapper = new CoreclrTestWrapperLib()%3B
+                  string testSubfolder = @"\$(Category)\$([System.String]::Copy('%(AllCMDs.RelativeDir)').Replace("$(_CMDDIR)$([System.IO.Path]::DirectorySeparatorChar)",''))"%3B
+                  outputFile = System.IO.Path.GetFullPath(globalVar.reportBase + testSubfolder + @"%(AllCMDs.FileName).output.txt")%3B
+                  errorFile = System.IO.Path.GetFullPath(globalVar.reportBase + testSubfolder + @"%(AllCMDs.FileName).error.txt")%3B
+                  testExecutable = System.IO.Path.GetFullPath(globalVar.testBinaryBase + @"$([System.String]::Copy('%(AllCMDs.FullPath)').Replace("$(_CMDDIR)",''))")%3B
+                  $(TestExecutableReplacement)
+                  outputDir = System.IO.Path.GetDirectoryName(outputFile)%3B
+
+                  if (!globalVar.runningInWindows) {
+                      testExecutable = testExecutable.Replace(".cmd", ".sh")%3B
+                  }
+
+                  System.IO.Directory.CreateDirectory(globalVar.reportBase + testSubfolder)%3B
+
+                  ret = wrapper.RunTest(testExecutable, outputFile, errorFile, globalVar.category, globalVar.testBinaryBase, outputDir)%3B
+                }
+                catch (Exception ex)
+                {
+                    infraEx = ex%3B
+                }
+
+                if (infraEx != null)
+                {
+                    Assert.True(false, "Test Infrastructure Failure: " + infraEx.ToString())%3B
+                }
+                else
+                {
+                    List<string> testOutput = new List<string>()%3B
+
+                    try
+                    {
+                        testOutput.AddRange(System.IO.File.ReadAllLines(errorFile))%3B
+                    }
+                    catch (Exception ex)
+                    {
+                        testOutput.Add("Unable to read error file: " + errorFile)%3B
+                        testOutput.Add(ex.ToString())%3B
+                    }
+
+                    testOutput.Add(string.Empty)%3B
+                    testOutput.Add("Return code:      " + ret)%3B
+                    testOutput.Add("Raw output file:      " + outputFile)%3B
+                    testOutput.Add("Raw output:")%3B
+
+                    try
+                    {
+                        testOutput.AddRange(System.IO.File.ReadAllLines(outputFile))%3B
+                    }
+                    catch(Exception ex)
+                    {
+                        testOutput.Add("Unable to read output file: " + outputFile)%3B
+                        testOutput.Add(ex.ToString())%3B
+                    }
+
+                    testOutput.Add("To run the test:")%3B
+                    testOutput.Add("> set CORE_ROOT=" + globalVar.coreRoot)%3B
+                    testOutput.Add("> " + testExecutable)%3B
+
+                    var unicodeControlCharsRegex = new Regex("%5C%5Cp{C}+")%3B
+
+                    // Remove all characters that have no visual or spatial representation.
+                    for (int i = 0%3B i < testOutput.Count%3B i++)
+                    {
+                        string line = testOutput[i]%3B
+                        line = unicodeControlCharsRegex.Replace(line, string.Empty)%3B
+                        testOutput[i] = line%3B
+                    }
+
+                    foreach (string line in testOutput)
+                    {
+                        output.WriteLine(line)%3B
+                    }
+
                     // Add Android app running log to testOutput
                     if (ret != CoreclrTestWrapperLib.EXIT_SUCCESS_CODE)
                     {
-                        string androidLogFile = System.IO.Path.Combine(outputDir, "adb-logcat-net.dot." + _Global.category + "-net.dot.MonoRunner.log")%3B
+                        string androidLogFile = System.IO.Path.Combine(outputDir, "adb-logcat-net.dot." + globalVar.category + "-net.dot.MonoRunner.log")%3B
                         if(File.Exists(androidLogFile))
                         {
                             testOutput.AddRange(System.IO.File.ReadAllLines(androidLogFile))%3B
@@ -383,7 +562,6 @@ namespace $([System.String]::Copy($(Category)).Replace(".","_").Replace("\","").
                 }
             }
         }
-
         ]]>
         </XUnitFact>
       </AllCommands>