Use ProcDump for dumps

Decided to go with procdump for getting our dumps.  While it's cleaner
and easier to create the dumps with a simple PInvoke call it's not a
complete solution.  The MiniDumpWriteDump API will always write the dump
in the architecture of the process calling the API.  Hence it would
create them for the architecture of RunTests, not the architecture of
the process being dumped.

In order to make it work I was going to have to buil two versions of
RunTests, or a simpler dumping program.  At that point it's just easier
to use ProcDump.  Even though I rather dislike an explicit download step
at the start of our CI runs.  Would be much nicer if sysinternal was
offered as NuGet package.

But the benefit here is clear so going forward with this.

BuildAndTest.proj

@@ -10,11 +10,12 @@
     <TestVsi Condition="'$(testVsiNetCore)' == 'true'">true</TestVsi>
     <Configuration Condition="'$(Configuration)' == ''">Debug</Configuration>
     <RunTestArgs Condition="'$(ManualTest)' == ''">$(RunTestArgs) -xml</RunTestArgs>
-    <RunTestArgs Condition="'$(Test64)' == 'true'">$(RunTestArgs) -test64</RunTestArgs>
-    <RunTestArgs Condition="'$(TestVsi)' == 'true'">$(RunTestArgs) -testVsi -timeout:5</RunTestArgs>
+    <RunTestArgs Condition="'$(Test64)' == 'true'">$(RunTestArgs) -test64 -timeout:2</RunTestArgs>
+    <RunTestArgs Condition="'$(TestVsi)' == 'true'">$(RunTestArgs) -testVsi -timeout:2</RunTestArgs>
     <RunTestArgs Condition="'$(TestVsiNetCore)' == 'true'">$(RunTestArgs) -trait:Feature=NetCore</RunTestArgs>
     <RunTestArgs Condition="'$(Trait)' != ''">$(RunTestArgs) -trait:$(Trait)</RunTestArgs>
     <RunTestArgs Condition="'$(NoTrait)' != ''">$(RunTestArgs) -notrait:$(NoTrait)</RunTestArgs>
+    <RunTestArgs Condition="'$(ProcDumpDir)' != ''">$(RunTestArgs) -procDumpPath:$(ProcDumpDir)</RunTestArgs>
     <DeployExtensionViaBuild Condition="'$(DeployExtensionViaBuild)' == ''">false</DeployExtensionViaBuild>
     <IncludePattern Condition="'$(IncludePattern)' == '' AND '$(TestVsi)' != 'true'">*.UnitTests.dll</IncludePattern>
     <IncludePattern Condition="'$(IncludePattern)' == '' AND '$(TestVsi)' == 'true'">*.IntegrationTests.dll</IncludePattern>

build/scripts/cibuild.ps1

@@ -52,6 +52,24 @@ function Terminate-BuildProcesses() {
     Get-Process vbcscompiler -ErrorAction SilentlyContinue | kill
 }
 
+# Ensure that procdump is available on the machine.  Returns the path to the directory that contains 
+# the procdump binaries (both 32 and 64 bit)
+function Ensure-ProcDump() {
+    $toolsDir = Join-Path $binariesDir "Tools"
+    $outDir = Join-Path $toolsDir "ProcDump"
+    $filePath = Join-Path $outDir "procdump.exe"
+    if (-not (Test-Path $filePath)) { 
+        Remove-Item -Re $filePath -ErrorAction SilentlyContinue
+        Create-Directory $outDir 
+        $zipFilePath = Join-Path $toolsDir "procdump.zip"
+        Invoke-WebRequest "https://download.sysinternals.com/files/Procdump.zip" -outfile $zipFilePath | Out-Null
+        Add-Type -AssemblyName System.IO.Compression.FileSystem
+        [IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $outDir)
+    }
+
+    return $outDir
+}
+
 # The Jenkins images used to execute our tests can live for a very long time.  Over the course
 # of hundreds of runs this can cause the %TEMP% folder to fill up.  To avoid this we redirect
 # %TEMP% into the binaries folder which is deleted at the end of every run as a part of cleaning
@@ -161,6 +179,7 @@ try {
     $testVsiArg = if ($testVsi) { "true" } else { "false" }
     $testVsiNetCoreArg = if ($testVsiNetCore) { "true" } else { "false" }
     $buildLog = Join-Path $binariesdir "Build.log"
+    $procDumpDir = Ensure-ProcDump
 
     # To help the VS SDK team track down their issues around install via build temporarily 
     # re-enabling the build based deployment
@@ -172,7 +191,7 @@ try {
         Write-Host "The testVsiNetCore option can't be combined with other test arguments"
     }
 
-    Run-MSBuild /p:BootstrapBuildPath="$bootstrapDir" BuildAndTest.proj /p:Configuration=$buildConfiguration /p:Test64=$test64Arg /p:TestVsi=$testVsiArg /p:TestDesktop=$testDesktop /p:TestCoreClr=$testCoreClr /p:TestVsiNetCore=$testVsiNetCoreArg /p:PathMap="$($repoDir)=q:\roslyn" /p:Feature=pdb-path-determinism /fileloggerparameters:LogFile="$buildLog"`;verbosity=diagnostic /p:DeployExtension=false /p:RoslynRuntimeIdentifier=win7-x64 /p:DeployExtensionViaBuild=$deployExtensionViaBuild /p:TreatWarningsAsErrors=true
+    Run-MSBuild /p:BootstrapBuildPath="$bootstrapDir" BuildAndTest.proj /p:Configuration=$buildConfiguration /p:Test64=$test64Arg /p:TestVsi=$testVsiArg /p:TestDesktop=$testDesktop /p:TestCoreClr=$testCoreClr /p:TestVsiNetCore=$testVsiNetCoreArg /p:PathMap="$($repoDir)=q:\roslyn" /p:Feature=pdb-path-determinism /fileloggerparameters:LogFile="$buildLog"`;verbosity=diagnostic /p:DeployExtension=false /p:RoslynRuntimeIdentifier=win7-x64 /p:DeployExtensionViaBuild=$deployExtensionViaBuild /p:TreatWarningsAsErrors=true /p:ProcDumpDir=$procDumpDir
 
     exit 0
 }

src/Tools/Source/RunTests/DumpUtil.cs

-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.IO;
-using System.Linq;
-using System.Runtime.InteropServices;
-using System.Text;
-using System.Threading.Tasks;
-using Microsoft.Win32.SafeHandles;
-
-namespace RunTests
-{
-    internal static class DumpUtil
-    {
-        [Flags]
-        private enum DumpType : uint
-        {
-            MiniDumpNormal = 0x00000000,
-            MiniDumpWithDataSegs = 0x00000001,
-            MiniDumpWithFullMemory = 0x00000002,
-            MiniDumpWithHandleData = 0x00000004,
-            MiniDumpFilterMemory = 0x00000008,
-            MiniDumpScanMemory = 0x00000010,
-            MiniDumpWithUnloadedModules = 0x00000020,
-            MiniDumpWithIndirectlyReferencedMemory = 0x00000040,
-            MiniDumpFilterModulePaths = 0x00000080,
-            MiniDumpWithProcessThreadData = 0x00000100,
-            MiniDumpWithPrivateReadWriteMemory = 0x00000200,
-            MiniDumpWithoutOptionalData = 0x00000400,
-            MiniDumpWithFullMemoryInfo = 0x00000800,
-            MiniDumpWithThreadInfo = 0x00001000,
-            MiniDumpWithCodeSegs = 0x00002000,
-            MiniDumpWithoutAuxiliaryState = 0x00004000,
-            MiniDumpWithFullAuxiliaryState = 0x00008000,
-            MiniDumpWithPrivateWriteCopyMemory = 0x00010000,
-            MiniDumpIgnoreInaccessibleMemory = 0x00020000,
-            MiniDumpValidTypeFlags = 0x0003ffff,
-        };
-
-        [DllImport("dbghelp.dll", EntryPoint = "MiniDumpWriteDump", CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Unicode, ExactSpelling = true, SetLastError = true)]
-        private static extern bool MiniDumpWriteDump(
-          IntPtr hProcess,
-          uint processId,
-          SafeFileHandle hFile,
-          uint dumpType,
-          IntPtr exceptionParam,
-          IntPtr userStreamParam,
-          IntPtr callbackParam);
-
-        internal static void WriteDump(Process process, string dumpFilePath)
-        {
-            using (var stream = File.Open(dumpFilePath, FileMode.Create, FileAccess.Write, FileShare.None))
-            {
-                var success = MiniDumpWriteDump(
-                    process.Handle,
-                    (uint)process.Id,
-                    stream.SafeFileHandle,
-                    (uint)(DumpType.MiniDumpNormal | DumpType.MiniDumpWithFullMemory | DumpType.MiniDumpWithFullMemoryInfo),
-                    IntPtr.Zero,
-                    IntPtr.Zero,
-                    IntPtr.Zero);
-                if (!success)
-                {
-                    throw new Exception($"Creating the minidump failed with error code {Marshal.GetLastWin32Error()}");
-                }
-            }
-        }
-    }
-}

src/Tools/Source/RunTests/Options.cs

@@ -63,6 +63,8 @@ internal class Options
         /// </summary>
         public TimeSpan? Timeout { get; set; }
 
+        public string ProcDumpPath { get; set; }
+
         public string XunitPath { get; set; }
 
         /// <summary>
@@ -163,6 +165,11 @@ bool isOption(string argument, string optionName, out string value)
 
                     index++;
                 }
+                else if (isOption(current, "-procdumpPath", out value))
+                {
+                    opt.ProcDumpPath = value;
+                    index++;
+                }
                 else
                 {
                     break;

src/Tools/Source/RunTests/ProcessUtil.cs

@@ -3,6 +3,7 @@
 using System.Diagnostics;
 using System.Linq;
 using System.Management;
+using System.Runtime.InteropServices;
 using System.Text;
 using System.Threading.Tasks;
 
@@ -77,5 +78,25 @@ internal static List<Process> GetProcessTree(Process process)
 
             return list;
         }
+
+        internal static bool Is64Bit(Process process)
+        {
+            if (Environment.GetEnvironmentVariable("PROCESSOR_ARCHITECTURE") == "x86")
+            {
+                return false;
+            }
+
+            bool isWow64;
+            if (!IsWow64Process(process.Handle, out isWow64))
+            {
+                throw new Exception($"{nameof(IsWow64Process)} failed with {Marshal.GetLastWin32Error()}");
+            }
+
+            return !isWow64;
+        }
+
+        [DllImport("kernel32.dll", SetLastError = true, CallingConvention = CallingConvention.Winapi)]
+        [return: MarshalAs(UnmanagedType.Bool)]
+        private static extern bool IsWow64Process([In] IntPtr process, [Out] out bool wow64Process);
     }
 }

src/Tools/Source/RunTests/Program.cs

@@ -152,21 +152,38 @@ private static void WriteLogFile(Options options)
         /// </summary>
         private static void HandleTimeout(Options options)
         {
-            Console.WriteLine("Roslyn Error: test timeout exceeded, dumping remaining processes");
-            var dumpDir = options.LogFilePath != null
-                ? Path.GetDirectoryName(options.LogFilePath)
-                : Directory.GetCurrentDirectory();
+            string GetProcDumpFilePath(Process proc) => ProcessUtil.Is64Bit(proc)
+                ? Path.Combine(options.ProcDumpPath, "procdump64.exe")
+                : Path.Combine(options.ProcDumpPath, "procdump.exe");
 
-            var counter = 0;
-            foreach (var proc in ProcessUtil.GetProcessTree(Process.GetCurrentProcess()).OrderBy(x => x.ProcessName))
+            void DumpProcess(Process targetProcess, string dumpFilePath)
             {
-                var dumpFilePath = Path.Combine(dumpDir, $"{proc.ProcessName}-{counter}.dmp");
-                counter++;
-                Console.Write($"Dumping {proc.ProcessName} {proc.Id} to {dumpFilePath} ... ");
+                Console.Write($"Dumping {targetProcess.ProcessName} {targetProcess.Id} to {dumpFilePath} ... ");
                 try
                 {
-                    DumpUtil.WriteDump(proc, dumpFilePath);
-                    Console.WriteLine("succeeded");
+                    var processStartInfo = new ProcessStartInfo();
+                    processStartInfo.FileName = GetProcDumpFilePath(targetProcess);
+                    processStartInfo.Arguments = $"-accepteula -ma {targetProcess.Id} {dumpFilePath}";
+                    processStartInfo.CreateNoWindow = true;
+                    processStartInfo.UseShellExecute = false;
+                    processStartInfo.WindowStyle = ProcessWindowStyle.Hidden;
+                    processStartInfo.RedirectStandardOutput = true;
+                    var process = Process.Start(processStartInfo);
+                    process.WaitForExit();
+
+                    // The exit code for procdump doesn't obey standard windows rules.  It will return non-zero
+                    // for succesful cases (possibly returning the count of dumps that were written).  Best 
+                    // backup is to test for the dump file being present.
+                    if (File.Exists(dumpFilePath))
+                    {
+                        Console.WriteLine("succeeded");
+                    }
+                    else
+                    {
+                        Console.WriteLine($"FAILED with {process.ExitCode}");
+                        Console.WriteLine($"{processStartInfo.FileName} {processStartInfo.Arguments}");
+                        Console.WriteLine(process.StandardOutput.ReadToEnd());
+                    }
                 }
                 catch (Exception ex)
                 {
@@ -175,6 +192,27 @@ private static void HandleTimeout(Options options)
                 }
             }
 
+            Console.WriteLine("Roslyn Error: test timeout exceeded, dumping remaining processes");
+
+            if (!string.IsNullOrEmpty(options.ProcDumpPath))
+            {
+                var dumpDir = options.LogFilePath != null
+                    ? Path.GetDirectoryName(options.LogFilePath)
+                    : Directory.GetCurrentDirectory();
+
+                var counter = 0;
+                foreach (var proc in ProcessUtil.GetProcessTree(Process.GetCurrentProcess()).OrderBy(x => x.ProcessName))
+                {
+                    var dumpFilePath = Path.Combine(dumpDir, $"{proc.ProcessName}-{counter}.dmp");
+                    DumpProcess(proc, dumpFilePath);
+                    counter++;
+                }
+            }
+            else
+            {
+                Console.WriteLine("Could not locate procdump");
+            }
+
             WriteLogFile(options);
         }
 

src/Tools/Source/RunTests/RunTests.csproj

@@ -49,7 +49,6 @@
     <Compile Include="Cache\WebDataStorage.Json.cs" />
     <Compile Include="Cache\WebDataStorage.cs" />
     <Compile Include="Constants.cs" />
-    <Compile Include="DumpUtil.cs" />
     <Compile Include="FileUtil.cs" />
     <Compile Include="Cache\IDataStorage.cs" />
     <Compile Include="ITestExecutor.cs" />