Dump tests that timeout in Jenkins

This contains two changes to our test runner:

1. Adds a -timeout value.  When that is exceeded dump files will be
written for the remaining processes and Runtests will exit with an
error.
1. Adds a sample timeout for our VSI tests.

BuildAndTest.proj

@@ -11,7 +11,7 @@
     <Configuration Condition="'$(Configuration)' == ''">Debug</Configuration>
     <RunTestArgs Condition="'$(ManualTest)' == ''">$(RunTestArgs) -xml</RunTestArgs>
     <RunTestArgs Condition="'$(Test64)' == 'true'">$(RunTestArgs) -test64</RunTestArgs>
-    <RunTestArgs Condition="'$(TestVsi)' == 'true'">$(RunTestArgs) -testVsi</RunTestArgs>
+    <RunTestArgs Condition="'$(TestVsi)' == 'true'">$(RunTestArgs) -testVsi -timeout:5</RunTestArgs>
     <RunTestArgs Condition="'$(TestVsiNetCore)' == 'true'">$(RunTestArgs) -trait:Feature=NetCore</RunTestArgs>
     <RunTestArgs Condition="'$(Trait)' != ''">$(RunTestArgs) -trait:$(Trait)</RunTestArgs>
     <RunTestArgs Condition="'$(NoTrait)' != ''">$(RunTestArgs) -notrait:$(NoTrait)</RunTestArgs>

src/Tools/Source/RunTests/CrashDumps.cs

-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.IO;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace RunTests
-{
-    public static class CrashDumps
-    {
-        public static bool TryMonitorProcess(Process processToMonitor, string outputPath)
-        {
-            var procDumpPath = TryGetProcDumpPath();
-
-            // Make sure everything is fully qualified as we are passing this to other processes
-            outputPath = Path.GetFullPath(outputPath);
-
-            if (procDumpPath == null)
-            {
-                return false;
-            }
-
-            var processStart = new ProcessStartInfo();
-
-            processStart.Arguments = GetProcDumpArgumentsForMonitoring(processToMonitor.Id, outputPath);
-            processStart.CreateNoWindow = true;
-            processStart.FileName = procDumpPath;
-            processStart.UseShellExecute = false;
-
-            var process = Process.Start(processStart);
-
-            return true;
-        }
-
-        private static string GetProcDumpArgumentsForMonitoring(int pid, string outputPath)
-        {
-            // Here's what these arguments mean:
-            //
-            // -g:  run as a native debugger in a managed process (no interop).
-            // -e:  dump when an unhandled exception happens
-            // -b:  dump when a breakpoint (__int 3 / Debugger.Break()) is encountered
-            // -ma: create a full memory dump
-            //
-            // without -g, procdump will not catch unhandled managed exception since CLR will always
-            // handle unhandled exception. for procdump point of view, there is no such thing as unhandled
-            // exception for managed app.
-
-            return $"-accepteula -g -e -b -ma {pid} {outputPath}";
-        }
-
-        private const string DotNetContinuousIntegrationProcDumpLocation = @"C:\Sysinternals\Procdump.exe";
-
-        private static string TryGetProcDumpPath()
-        {
-            if (File.Exists(DotNetContinuousIntegrationProcDumpLocation))
-            {
-                return DotNetContinuousIntegrationProcDumpLocation;
-            }
-
-            return null;
-        }
-    }
-}

src/Tools/Source/RunTests/DumpUtil.cs

+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Runtime.InteropServices;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace RunTests
+{
+    internal static class DumpUtil
+    {
+        [Flags]
+        private enum DumpType : uint
+        {
+            MiniDumpNormal = 0x00000000,
+            MiniDumpWithDataSegs = 0x00000001,
+            MiniDumpWithFullMemory = 0x00000002,
+            MiniDumpWithHandleData = 0x00000004,
+            MiniDumpFilterMemory = 0x00000008,
+            MiniDumpScanMemory = 0x00000010,
+            MiniDumpWithUnloadedModules = 0x00000020,
+            MiniDumpWithIndirectlyReferencedMemory = 0x00000040,
+            MiniDumpFilterModulePaths = 0x00000080,
+            MiniDumpWithProcessThreadData = 0x00000100,
+            MiniDumpWithPrivateReadWriteMemory = 0x00000200,
+            MiniDumpWithoutOptionalData = 0x00000400,
+            MiniDumpWithFullMemoryInfo = 0x00000800,
+            MiniDumpWithThreadInfo = 0x00001000,
+            MiniDumpWithCodeSegs = 0x00002000,
+            MiniDumpWithoutAuxiliaryState = 0x00004000,
+            MiniDumpWithFullAuxiliaryState = 0x00008000,
+            MiniDumpWithPrivateWriteCopyMemory = 0x00010000,
+            MiniDumpIgnoreInaccessibleMemory = 0x00020000,
+            MiniDumpValidTypeFlags = 0x0003ffff,
+        };
+
+        [DllImport("dbghelp.dll", EntryPoint = "MiniDumpWriteDump", CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Unicode, ExactSpelling = true, SetLastError = true)]
+        private static extern bool MiniDumpWriteDump(
+          IntPtr hProcess,
+          uint processId,
+          IntPtr hFile,
+          uint dumpType,
+          IntPtr exceptionParam,
+          IntPtr userStreamParam,
+          IntPtr callbackParam);
+
+        internal static void WriteDump(Process process, string dumpFilePath)
+        {
+            using (var stream = File.Open(dumpFilePath, FileMode.Create, FileAccess.Write, FileShare.None))
+            {
+                var success = MiniDumpWriteDump(
+                    process.Handle,
+                    (uint)process.Id,
+                    stream.SafeFileHandle.DangerousGetHandle(),
+                    (uint)(DumpType.MiniDumpNormal | DumpType.MiniDumpWithFullMemory | DumpType.MiniDumpWithFullMemoryInfo),
+                    IntPtr.Zero,
+                    IntPtr.Zero,
+                    IntPtr.Zero);
+                if (!success)
+                {
+                    throw new Exception("Could not create dump");
+                }
+            }
+        }
+    }
+}

src/Tools/Source/RunTests/Options.cs

@@ -58,6 +58,11 @@ internal class Options
         /// </summary>
         public List<string> Assemblies { get; set; }
 
+        /// <summary>
+        /// Time after which the runner should kill the xunit process and exit with a failure.
+        /// </summary>
+        public TimeSpan? Timeout { get; set; }
+
         public string XunitPath { get; set; }
 
         /// <summary>
@@ -144,6 +149,20 @@ bool isOption(string argument, string optionName, out string value)
                     opt.NoTrait = value;
                     index++;
                 }
+                else if (isOption(current, "-timeout", out value))
+                {
+                    if (int.TryParse(value, out var minutes))
+                    {
+                        opt.Timeout = TimeSpan.FromMinutes(minutes);
+                    }
+                    else
+                    {
+                        Console.WriteLine($"{value} is not a valid minute value for timeout");
+                        allGood = false;
+                    }
+
+                    index++;
+                }
                 else
                 {
                     break;

src/Tools/Source/RunTests/ProcessRunner.cs

@@ -11,30 +11,15 @@ namespace RunTests
 {
     public sealed class ProcessOutput
     {
-        private readonly int _exitCode;
-        private readonly IList<string> _outputLines;
-        private readonly IList<string> _errorLines;
-
-        public int ExitCode
-        {
-            get { return _exitCode; }
-        }
-
-        public IList<string> OutputLines
-        {
-            get { return _outputLines; }
-        }
-
-        public IList<string> ErrorLines
-        {
-            get { return _errorLines; }
-        }
+        public int ExitCode { get; }
+        public IList<string> OutputLines { get; }
+        public IList<string> ErrorLines { get; }
 
         public ProcessOutput(int exitCode, IList<string> outputLines, IList<string> errorLines)
         {
-            _exitCode = exitCode;
-            _outputLines = outputLines;
-            _errorLines = errorLines;
+            ExitCode = exitCode;
+            OutputLines = outputLines;
+            ErrorLines = errorLines;
         }
     }
 
@@ -56,60 +41,27 @@ public static void OpenFile(string file)
             string workingDirectory = null,
             bool captureOutput = false,
             bool displayWindow = true,
-            Dictionary<string, string> environmentVariables = null,
-            Action<Process> processMonitor = null)
+            Dictionary<string, string> environmentVariables = null)
         {
             cancellationToken.ThrowIfCancellationRequested();
 
             var taskCompletionSource = new TaskCompletionSource<ProcessOutput>();
-
-            var process = new Process();
-            process.EnableRaisingEvents = true;
-            process.StartInfo = CreateProcessStartInfo(executable, arguments, workingDirectory, captureOutput, displayWindow);
-
-            var task = CreateTask(process, taskCompletionSource, cancellationToken);
-
-            process.Start();
-
-            processMonitor?.Invoke(process);
-
-            if (lowPriority)
-            {
-                process.PriorityClass = ProcessPriorityClass.BelowNormal;
-            }
-
-            if (process.StartInfo.RedirectStandardOutput)
-            {
-                process.BeginOutputReadLine();
-            }
-
-            if (process.StartInfo.RedirectStandardError)
-            {
-                process.BeginErrorReadLine();
-            }
-
-            return task;
+            var processStartInfo = CreateProcessStartInfo(executable, arguments, workingDirectory, captureOutput, displayWindow);
+            return CreateProcessTask(processStartInfo, lowPriority, taskCompletionSource, cancellationToken);
         }
 
-        private static Task<ProcessOutput> CreateTask(
-            Process process,
+        private static Task<ProcessOutput> CreateProcessTask(
+            ProcessStartInfo processStartInfo,
+            bool lowPriority,
             TaskCompletionSource<ProcessOutput> taskCompletionSource,
             CancellationToken cancellationToken)
         {
-            cancellationToken.ThrowIfCancellationRequested();
-
-            if (taskCompletionSource == null)
-            {
-                throw new ArgumentException("taskCompletionSource");
-            }
-
-            if (process == null)
-            {
-                return taskCompletionSource.Task;
-            }
-
             var errorLines = new List<string>();
             var outputLines = new List<string>();
+            var process = new Process();
+
+            process.EnableRaisingEvents = true;
+            process.StartInfo = processStartInfo;
 
             process.OutputDataReceived += (s, e) =>
                 {
@@ -159,6 +111,23 @@ public static void OpenFile(string file)
                     }
                 });
 
+            process.Start();
+
+            if (lowPriority)
+            {
+                process.PriorityClass = ProcessPriorityClass.BelowNormal;
+            }
+
+            if (processStartInfo.RedirectStandardOutput)
+            {
+                process.BeginOutputReadLine();
+            }
+
+            if (processStartInfo.RedirectStandardError)
+            {
+                process.BeginErrorReadLine();
+            }
+
             return taskCompletionSource.Task;
         }
 

src/Tools/Source/RunTests/ProcessTestExecutor.cs

@@ -91,8 +91,7 @@ public async Task<TestResult> RunTestAsync(AssemblyInfo assemblyInfo, Cancellati
                     lowPriority: false,
                     displayWindow: false,
                     captureOutput: true,
-                    cancellationToken: cancellationToken,
-                    processMonitor: p => CrashDumps.TryMonitorProcess(p, dumpOutputFilePath)).ConfigureAwait(false);
+                    cancellationToken: cancellationToken);
                 var span = DateTime.UtcNow - start;
 
                 if (processOutput.ExitCode != 0)

src/Tools/Source/RunTests/ProcessUtil.cs

+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Management;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace RunTests
+{
+    internal static class ProcessUtil
+    {
+        internal static int? TryGetParentProcessId(Process p)
+        {
+            try
+            {
+                ManagementObject mo = new ManagementObject("win32_process.handle='" + p.Id + "'");
+                mo.Get();
+                return Convert.ToInt32(mo["ParentProcessId"]);
+            }
+            catch 
+            {
+                return null;
+            }
+        }
+
+        /// <summary>
+        /// Return the list of processes which are direct children of the provided <paramref name="process"/> 
+        /// instance.
+        /// </summary>
+        /// <remarks>
+        /// This is a best effort API.  It can be thwarted by process instances starting / stopping during
+        /// the building of this list.
+        /// </remarks>
+        internal static List<Process> GetProcessChildren(Process process) => GetProcessChildrenCore(process, Process.GetProcesses());
+
+        private static List<Process> GetProcessChildrenCore(Process parentProcess, IEnumerable<Process> processes)
+        {
+            var list = new List<Process>();
+            foreach (var process in processes)
+            {
+                var parentId = TryGetParentProcessId(process);
+                if (parentId == parentProcess.Id)
+                {
+                    list.Add(process);
+                }
+            }
+
+            return list;
+        }
+
+        /// <summary>
+        /// Return the list of processes which are direct or indirect children of the provided <paramref name="process"/> 
+        /// instance.
+        /// </summary>
+        /// <remarks>
+        /// This is a best effort API.  It can be thwarted by process instances starting / stopping during
+        /// the building of this list.
+        /// </remarks>
+        internal static List<Process> GetProcessTree(Process process)
+        {
+            var processes = Process.GetProcesses();
+            var list = new List<Process>();
+            var toVisit = new Queue<Process>();
+            toVisit.Enqueue(process);
+
+            while (toVisit.Count > 0)
+            {
+                var cur = toVisit.Dequeue();
+                var children = GetProcessChildrenCore(cur, processes);
+                foreach (var child in children)
+                {
+                    toVisit.Enqueue(child);
+                    list.Add(child);
+                }
+            }
+
+            return list;
+        }
+    }
+}

src/Tools/Source/RunTests/Program.cs

@@ -14,6 +14,7 @@
 using System.Collections.Immutable;
 using Newtonsoft.Json;
 using System.Reflection;
+using System.Diagnostics;
 
 namespace RunTests
 {
@@ -41,7 +42,46 @@ internal static int Main(string[] args)
                 cts.Cancel();
             };
 
-            return RunCore(options, cts.Token).GetAwaiter().GetResult();
+            return Run(options, cts.Token).GetAwaiter().GetResult();
+        }
+
+        private static async Task<int> Run(Options options, CancellationToken cancellationToken)
+        {
+            if (options.Timeout == null)
+            {
+                return await RunCore(options, cancellationToken);
+            }
+
+            var timeoutTask = Task.Delay(options.Timeout.Value);
+            var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+            var runTask = RunCore(options, cts.Token);
+
+            if (cancellationToken.IsCancellationRequested)
+            {
+                return ExitFailure;
+            }
+
+            var finishedTask = await Task.WhenAny(timeoutTask, runTask);
+            if (finishedTask == timeoutTask)
+            {
+                HandleTimeout(options);
+                cts.Cancel();
+
+                try
+                {
+                    // Need to await here to ensure that all of the child processes are properly 
+                    // killed before we exit.
+                    await runTask;
+                }
+                catch
+                {
+                    // Cancellation exceptions expected here. 
+                }
+
+                return ExitFailure;
+            }
+
+            return await runTask;
         }
 
         private static async Task<int> RunCore(Options options, CancellationToken cancellationToken)
@@ -106,6 +146,29 @@ private static void WriteLogFile(Options options)
             Logger.Clear();
         }
 
+        /// <summary>
+        /// Invoked when a timeout occurs and we need to dump all of the test processes and shut down 
+        /// the runnner.
+        /// </summary>
+        private static void HandleTimeout(Options options)
+        {
+            Console.WriteLine("Roslyn Error: test timeout exceeded, dumping remaining processes");
+            var dumpDir = options.LogFilePath != null
+                ? Path.GetDirectoryName(options.LogFilePath)
+                : Directory.GetCurrentDirectory();
+
+            var counter = 0;
+            foreach (var proc in ProcessUtil.GetProcessTree(Process.GetCurrentProcess()).OrderBy(x => x.ProcessName))
+            {
+                var dumpFilePath = Path.Combine(dumpDir, $"{proc.ProcessName}-{counter}.dmp");
+                counter++;
+                Console.WriteLine($"Dumping {proc.ProcessName} {proc.Id} to {dumpFilePath}");
+                DumpUtil.WriteDump(proc, dumpFilePath);
+            }
+
+            WriteLogFile(options);
+        }
+
         /// <summary>
         /// Quick sanity check to look over the set of assemblies to make sure they are valid and something was
         /// specified.

src/Tools/Source/RunTests/RunTests.csproj

@@ -20,6 +20,7 @@
   <ItemGroup>
     <Reference Include="System" />
     <Reference Include="System.Core" />
+    <Reference Include="System.Management" />
     <Reference Include="System.Xml.Linq" />
     <Reference Include="System.Xml" />
     <PackageReference Include="Newtonsoft.Json">
@@ -48,7 +49,7 @@
     <Compile Include="Cache\WebDataStorage.Json.cs" />
     <Compile Include="Cache\WebDataStorage.cs" />
     <Compile Include="Constants.cs" />
-    <Compile Include="CrashDumps.cs" />
+    <Compile Include="DumpUtil.cs" />
     <Compile Include="FileUtil.cs" />
     <Compile Include="Cache\IDataStorage.cs" />
     <Compile Include="ITestExecutor.cs" />
@@ -57,6 +58,7 @@
     <Compile Include="Options.cs" />
     <Compile Include="ProcessRunner.cs" />
     <Compile Include="ProcessTestExecutor.cs" />
+    <Compile Include="ProcessUtil.cs" />
     <Compile Include="Program.cs" />
     <Compile Include="TestRunner.cs" />
     <Compile Include="ConsoleUtil.cs" />

src/Tools/Source/RunTests/TestRunner.cs

@@ -103,7 +103,7 @@ internal async Task<RunAllResult> RunAllAsync(IEnumerable<AssemblyInfo> assembly
                     Console.Write($", {failures, 2} failures");
                 }
                 Console.WriteLine();
-                Task.WaitAny(running.ToArray());
+                await Task.WhenAny(running.ToArray());
             } while (running.Count > 0);
 
             Print(completed);