Program for running tests in parallel

Currently tests are run on the developers machine using

> msbuild BuildAndTest.proj

This takes ~25 minutes to complete on the CI machine and ~45 minutes on
a developer machine (developer machines run a greater number of suites).
This is not conductive to quick bug fixes.

Ideally the infrastructure would switch over to using xunit 2 which can
handle parallelizing the tests for us.  Unfortunately this isn't an
option because:

1. The 32 bit runner quickly OOMS on our suites
2. A number of our tests can't run in 64 bit

Long term xunit2 is still our goal.  Short term though this tool will
use process level parallelization to run our suites.

BuildAndTest.proj

@@ -12,6 +12,7 @@
   <PropertyGroup>
     <RoslynSolution>$(MSBuildThisFileDirectory)\Src\Roslyn.sln</RoslynSolution>
     <RoslynSolution Condition="$(CIBuild) == 'true'">$(MSBuildThisFileDirectory)\Src\Roslyn2013.sln</RoslynSolution>
+    <Configuration Condition="'$(Configuration)' == ''">Debug</Configuration>
   </PropertyGroup>
 
   <Target Name="RestorePackages">
@@ -45,7 +46,7 @@
     <ItemGroup Condition="'$(CIBuild)' == ''">
       <TestAssemblies 
         Include="Binaries\$(Configuration)\**\*.UnitTests*.dll" 
-        Exclude="Binaries\Roslyn.Compilers.NativeClient.UnitTests.dll" />
+        Exclude="Binaries\$(Configuration)\Roslyn.Compilers.NativeClient.UnitTests.dll" />
     </ItemGroup>
 
     <ItemGroup Condition="'$(CIBuild)' == 'true'">
@@ -75,10 +76,8 @@
       <TestAssemblies Include="Binaries\Debug\Roslyn.Services.VisualBasic.UnitTests.dll" />
     </ItemGroup>
 
-    <xunit
-      Assemblies="@(TestAssemblies)"
-      Html="UnitTestResults.html"
-      ShadowCopy="false" />
+    <Exec Command="Binaries\$(Configuration)\RunTests.exe packages\xunit.runners.2.0.0-rc1-build2826\tools\xunit.console.x86.exe @(TestAssemblies, ' ')" />
+
   </Target>
 
   <Target Name="BuildAndTest"

src/Roslyn.sln

@@ -149,6 +149,8 @@ Project("{F184B08F-C81C-45F6-A57F-5ABD9991F28F}") = "BasicWorkspace.Desktop", "W
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FakeSign", "Tools\Source\FakeSign\FakeSign.csproj", "{97CC7ABF-7E07-4F3A-947B-8C2D8F916450}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RunTests", "Tools\Source\RunTests\RunTests.csproj", "{1A3941F1-1E1F-4EF7-8064-7729C4C2E2AA}"
+EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenSourceDebug", "Tools\Source\OpenSourceDebug\OpenSourceDebug.csproj", "{43026D51-3083-4850-928D-07E1883D5B1A}"
 EndProject
 Project("{F184B08F-C81C-45F6-A57F-5ABD9991F28F}") = "BasicFeatures", "Features\VisualBasic\BasicFeatures.vbproj", "{A1BCD0CE-6C2F-4F8C-9A48-D9D93928E26D}"
@@ -764,6 +766,14 @@ Global
 		{97CC7ABF-7E07-4F3A-947B-8C2D8F916450}.Release|Any CPU.Build.0 = Release|Any CPU
 		{97CC7ABF-7E07-4F3A-947B-8C2D8F916450}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
 		{97CC7ABF-7E07-4F3A-947B-8C2D8F916450}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+		{1A3941F1-1E1F-4EF7-8064-7729C4C2E2AA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{1A3941F1-1E1F-4EF7-8064-7729C4C2E2AA}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{1A3941F1-1E1F-4EF7-8064-7729C4C2E2AA}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+		{1A3941F1-1E1F-4EF7-8064-7729C4C2E2AA}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+		{1A3941F1-1E1F-4EF7-8064-7729C4C2E2AA}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{1A3941F1-1E1F-4EF7-8064-7729C4C2E2AA}.Release|Any CPU.Build.0 = Release|Any CPU
+		{1A3941F1-1E1F-4EF7-8064-7729C4C2E2AA}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+		{1A3941F1-1E1F-4EF7-8064-7729C4C2E2AA}.Release|Mixed Platforms.Build.0 = Release|Any CPU
 		{43026D51-3083-4850-928D-07E1883D5B1A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{43026D51-3083-4850-928D-07E1883D5B1A}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{43026D51-3083-4850-928D-07E1883D5B1A}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU

src/Roslyn2013.sln

@@ -163,6 +163,8 @@ Project("{F184B08F-C81C-45F6-A57F-5ABD9991F28F}") = "BasicWorkspace.Desktop", "W
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FakeSign", "Tools\Source\FakeSign\FakeSign.csproj", "{97CC7ABF-7E07-4F3A-947B-8C2D8F916450}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RunTests", "Tools\Source\RunTests\RunTests.csproj", "{1A3941F1-1E1F-4EF7-8064-7729C4C2E2AA}"
+EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenSourceDebug", "Tools\Source\OpenSourceDebug\OpenSourceDebug.csproj", "{43026D51-3083-4850-928D-07E1883D5B1A}"
 EndProject
 Project("{F184B08F-C81C-45F6-A57F-5ABD9991F28F}") = "BasicFeatures", "Features\VisualBasic\BasicFeatures.vbproj", "{A1BCD0CE-6C2F-4F8C-9A48-D9D93928E26D}"
@@ -1088,6 +1090,14 @@ Global
 		{97CC7ABF-7E07-4F3A-947B-8C2D8F916450}.Release|Win32.Build.0 = Release|Any CPU
 		{97CC7ABF-7E07-4F3A-947B-8C2D8F916450}.Release|x86.ActiveCfg = Release|Any CPU
 		{97CC7ABF-7E07-4F3A-947B-8C2D8F916450}.Release|x86.Build.0 = Release|Any CPU
+		{1A3941F1-1E1F-4EF7-8064-7729C4C2E2AA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{1A3941F1-1E1F-4EF7-8064-7729C4C2E2AA}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{1A3941F1-1E1F-4EF7-8064-7729C4C2E2AA}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+		{1A3941F1-1E1F-4EF7-8064-7729C4C2E2AA}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+		{1A3941F1-1E1F-4EF7-8064-7729C4C2E2AA}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{1A3941F1-1E1F-4EF7-8064-7729C4C2E2AA}.Release|Any CPU.Build.0 = Release|Any CPU
+		{1A3941F1-1E1F-4EF7-8064-7729C4C2E2AA}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+		{1A3941F1-1E1F-4EF7-8064-7729C4C2E2AA}.Release|Mixed Platforms.Build.0 = Release|Any CPU
 		{43026D51-3083-4850-928D-07E1883D5B1A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{43026D51-3083-4850-928D-07E1883D5B1A}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{43026D51-3083-4850-928D-07E1883D5B1A}.Debug|ARM.ActiveCfg = Debug|Any CPU

src/Tools/Source/RunTests/ConsoleUtil.cs

+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace RunTests
+{
+    internal static class ConsoleUtil
+    {
+        internal static void Write(ConsoleColor color, string format, params object[] args)
+        {
+            WithColor(color, () => Console.Write(format, args));
+        }
+
+        internal static void WriteLine(ConsoleColor color, string format, params object[] args)
+        {
+            WithColor(color, () => Console.WriteLine(format, args));
+        }
+
+        private static void WithColor(ConsoleColor color, Action action)
+        {
+            var saved = Console.ForegroundColor;
+            try
+            {
+                Console.Out.Flush();
+                Console.ForegroundColor = color;
+                action();
+                Console.Out.Flush();
+            }
+            finally
+            {
+                Console.ForegroundColor = saved;
+            }
+        }
+    }
+}

src/Tools/Source/RunTests/ProcessRunner.cs

+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace RunTests
+{
+    public sealed class ProcessOutput
+    {
+        private readonly int exitCode;
+        private readonly IEnumerable<string> outputLines;
+        private readonly IEnumerable<string> errorLines;
+
+        public ProcessOutput(int exitCode, IEnumerable<string> outputLines, IEnumerable<string> errorLines)
+        {
+            this.exitCode = exitCode;
+            this.outputLines = outputLines;
+            this.errorLines = errorLines;
+        }
+
+        public int ExitCode { get { return exitCode; } }
+
+        public IEnumerable<string> OutputLines
+        {
+            get { return outputLines; }
+        }
+
+        public IEnumerable<string> ErrorLines
+        {
+            get { return errorLines; }
+        }
+    }
+
+    public static class ProcessRunner
+    {
+        public static void OpenFile(string file)
+        {
+            if (File.Exists(file))
+            {
+                Process.Start(file);
+            }
+        }
+
+        public static Task<ProcessOutput> RunProcessAsync(
+            string executable,
+            string arguments,
+            bool lowPriority,
+            CancellationToken cancellationToken,
+            string workingDirectory = null,
+            bool captureOutput = false,
+            bool displayWindow = true,
+            bool elevated = false,
+            Dictionary<string, string> environmentVariables = null)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            var taskCompletionSource = new TaskCompletionSource<ProcessOutput>();
+
+            var process = new Process();
+            process.EnableRaisingEvents = true;
+            if (elevated)
+            {
+                process.StartInfo = CreateElevatedStartInfo(executable, arguments, workingDirectory, captureOutput, displayWindow);
+            }
+            else
+            {
+                process.StartInfo = CreateProcessStartInfo(executable, arguments, workingDirectory, captureOutput, displayWindow);
+            }
+
+            var task = CreateTask(process, taskCompletionSource, cancellationToken);
+
+            process.Start();
+
+            if (lowPriority)
+            {
+                process.PriorityClass = ProcessPriorityClass.BelowNormal;
+            }
+
+            if (process.StartInfo.RedirectStandardOutput)
+            {
+                process.BeginOutputReadLine();
+            }
+
+            if (process.StartInfo.RedirectStandardError)
+            {
+                process.BeginErrorReadLine();
+            }
+
+            return task;
+        }
+
+        private static Task<ProcessOutput> CreateTask(
+            Process process,
+            TaskCompletionSource<ProcessOutput> taskCompletionSource,
+            CancellationToken cancellationToken)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            if (taskCompletionSource == null)
+            {
+                throw new ArgumentException("taskCompletionSource");
+            }
+
+            if (process == null)
+            {
+                return taskCompletionSource.Task;
+            }
+
+            var errorLines = new List<string>();
+            var outputLines = new List<string>();
+
+            process.OutputDataReceived += (s, e) =>
+                {
+                    if (e.Data != null)
+                    {
+                        outputLines.Add(e.Data);
+                    }
+                };
+
+            process.ErrorDataReceived += (s, e) =>
+                {
+                    if (e.Data != null)
+                    {
+                        errorLines.Add(e.Data);
+                    }
+                };
+
+            process.Exited += (s, e) =>
+                {
+                    var processOutput = new ProcessOutput(process.ExitCode, outputLines, errorLines);
+                    taskCompletionSource.TrySetResult(processOutput);
+                };
+
+            var registration = cancellationToken.Register(() =>
+                {
+                    if (taskCompletionSource.TrySetCanceled())
+                    {
+                        // If the underlying process is still running, we should kill it
+                        if (!process.HasExited)
+                        {
+                            try
+                            {
+                                process.Kill();
+                            }
+                            catch (InvalidOperationException)
+                            {
+                                // Ignore, since the process is already dead
+                            }
+                        }
+                    }
+                });
+
+            return taskCompletionSource.Task;
+        }
+
+        private static ProcessStartInfo CreateProcessStartInfo(
+            string executable, string arguments,
+            string workingDirectory,
+            bool captureOutput,
+            bool displayWindow,
+            Dictionary<string, string> environmentVariables = null)
+        {
+            var processStartInfo = new ProcessStartInfo(executable, arguments);
+
+            if (!string.IsNullOrEmpty(workingDirectory))
+            {
+                processStartInfo.WorkingDirectory = workingDirectory;
+            }
+
+            if (environmentVariables != null)
+            {
+                foreach (var pair in environmentVariables)
+                {
+                    processStartInfo.EnvironmentVariables[pair.Key] = pair.Value;
+                }
+            }
+
+            if (captureOutput)
+            {
+                processStartInfo.UseShellExecute = false;
+                processStartInfo.RedirectStandardOutput = true;
+                processStartInfo.RedirectStandardError = true;
+            }
+            else
+            {
+                processStartInfo.CreateNoWindow = !displayWindow;
+                processStartInfo.UseShellExecute = displayWindow;
+            }
+
+            return processStartInfo;
+        }
+
+        public static ProcessStartInfo CreateElevatedStartInfo(string executable, string arguments, string workingDirectory, bool captureOutput, bool displayWindow)
+        {
+            var adminInfo = new ProcessStartInfo(executable, arguments);
+            adminInfo.WindowStyle = ProcessWindowStyle.Hidden;
+            adminInfo.CreateNoWindow = true;
+            adminInfo.Verb = "runas";
+
+            if (!string.IsNullOrEmpty(workingDirectory))
+            {
+                adminInfo.WorkingDirectory = workingDirectory;
+            }
+
+            return adminInfo;
+        }
+
+        public static bool WaitForProcessExit(string procName, int timeoutInSeconds = 300)
+        {
+            int count = 0;
+            var procs = Process.GetProcessesByName(procName);
+            while (procs.Length > 0)
+            {
+                Thread.Sleep(1000);
+                procs = Process.GetProcessesByName(procName);
+                count++;
+                if (count > timeoutInSeconds)
+                {
+                    return false;
+                }
+            }
+
+            return true;
+        }
+    }
+}

src/Tools/Source/RunTests/Program.cs

+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace RunTests
+{
+    internal sealed class Program
+    {
+        internal static int Main(string[] args)
+        {
+            if (args.Length < 2)
+            {
+                PrintUsage();
+                return 1;
+            }
+
+            var xunit = args[0];
+            var list = new List<string>(args.Skip(1));
+            var testRunner = new TestRunner(xunit);
+            var start = DateTime.Now;
+            Console.WriteLine("Running {0} tests", list.Count);
+            var result = testRunner.RunAll(list).Result;
+            var span = DateTime.Now - start;
+            if (!result)
+            {
+                ConsoleUtil.WriteLine(ConsoleColor.Red, "Test failures encountered: {0}", span);
+                return 1;
+            }
+
+            Console.WriteLine("All tests passed: {0}", span);
+            return 0;
+        }
+
+        private static void PrintUsage()
+        {
+            Console.WriteLine("runtests [xunit-console-runner] [assembly1] [assembly2] [...]");
+        }
+    }
+}

src/Tools/Source/RunTests/RunTests.csproj

+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ImportGroup Label="Settings">
+    <Import Project="..\..\..\Tools\Microsoft.CodeAnalysis.Toolset.Open\Targets\VSL.Settings.targets" />
+    <Import Project="..\..\..\..\build\VSL.Settings.Closed.targets" />
+  </ImportGroup>
+  <PropertyGroup>
+    <NonShipping>True</NonShipping>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{1A3941F1-1E1F-4EF7-8064-7729C4C2E2AA}</ProjectGuid>
+    <OutputType>Exe</OutputType>
+    <RootNamespace>RunTests</RootNamespace>
+    <AssemblyName>RunTests</AssemblyName>
+    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' " />
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' " />
+  <ItemGroup>
+    <Reference Include="System" />
+    <Reference Include="System.Core" />
+    <Reference Include="System.Xml.Linq" />
+    <Reference Include="System.Xml" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="ProcessRunner.cs" />
+    <Compile Include="Program.cs" />
+    <Compile Include="TestRunner.cs" />
+    <Compile Include="ConsoleUtil.cs" />
+  </ItemGroup>
+  <ImportGroup Label="Targets">
+    <Import Project="..\..\..\Tools\Microsoft.CodeAnalysis.Toolset.Open\Targets\VSL.Imports.targets" />
+    <Import Project="..\..\..\..\build\VSL.Imports.Closed.targets" />
+    <Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" />
+  </ImportGroup>
+</Project>

src/Tools/Source/RunTests/TestRunner.cs

+using System;
+using System.Collections.Generic;
+using System.Collections.ObjectModel;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace RunTests
+{
+    internal sealed class TestRunner
+    {
+        private struct TestResult
+        {
+            internal readonly bool Succeeded;
+            internal readonly string AssemblyName;
+            internal readonly TimeSpan TimeSpan;
+
+            internal TestResult(bool succeeded, string assemblyName, TimeSpan timeSpan)
+            {
+                Succeeded = succeeded;
+                AssemblyName = assemblyName;
+                TimeSpan = timeSpan;
+            }
+        }
+
+        private readonly string _xunitConsolePath;
+
+        internal TestRunner(string xunitConsolePath)
+        {
+            _xunitConsolePath = xunitConsolePath;
+        }
+
+        internal async Task<bool> RunAll(IEnumerable<string> assemblyList)
+        {
+            var max = Environment.ProcessorCount;
+            var allPassed = true;
+            var waiting = new Stack<string>(assemblyList); 
+            var running = new List<Task<TestResult>>();
+            var completed = new List<TestResult>();
+
+            do
+            {
+                var i = 0;
+                while (i < running.Count)
+                {
+                    var task = running[i];
+                    if (task.IsCompleted)
+                    {
+                        var testResult = await task.ConfigureAwait(false);
+                        if (!testResult.Succeeded)
+                        {
+                            allPassed = false;
+                        }
+
+                        completed.Add(testResult);
+                        running.RemoveAt(i);
+                    }
+                    else
+                    {
+                        i++;
+                    }
+                }
+
+                while (running.Count < max && waiting.Count > 0)
+                {
+                    var task = RunTest(waiting.Pop());
+                    running.Add(task);
+                }
+
+                Console.WriteLine("  {0} running, {1} queued, {2} completed", running.Count, waiting.Count, completed.Count);
+                Task.WaitAny(running.ToArray());
+            } while (running.Count > 0);
+
+            Print(completed);
+
+            return allPassed;
+        }
+
+        private void Print(List<TestResult> testResults)
+        {
+            testResults.Sort((x, y) => x.AssemblyName.CompareTo(y.AssemblyName));
+
+            Console.WriteLine("================");
+            foreach (var testResult in testResults)
+            {
+                var color = testResult.Succeeded ? Console.ForegroundColor : ConsoleColor.Red;
+                ConsoleUtil.WriteLine(color, "{0,-75} {1} {2}", testResult.AssemblyName, testResult.Succeeded ? "PASSED" : "FAILED", testResult.TimeSpan);
+            }
+            Console.WriteLine("================");
+        }
+
+        private async Task<TestResult> RunTest(string assemblyPath)
+        {
+            var assemblyName = Path.GetFileName(assemblyPath);
+            var resultsPath = Path.Combine(Path.GetDirectoryName(assemblyPath), Path.ChangeExtension(assemblyName, ".html"));
+            var builder = new StringBuilder();
+            builder.AppendFormat(@"""{0}""", assemblyPath);
+            builder.AppendFormat(@" -html ""{0}""", resultsPath);
+            builder.Append(" -noshadow");
+
+            var start = DateTime.Now;
+            var processOutput = await ProcessRunner.RunProcessAsync(_xunitConsolePath, builder.ToString(), lowPriority: false, displayWindow: false, captureOutput: true, cancellationToken: CancellationToken.None).ConfigureAwait(false);
+            var span = DateTime.Now - start;
+
+            if (processOutput.ExitCode != 0)
+            {
+                // On occasion we get a non-0 output but no actual data in the result file.  Switch to output in this 
+                // case.
+                var all = File.ReadAllText(resultsPath).Trim();
+                if (all.Length == 0)
+                {
+                    var output = processOutput.OutputLines.Concat(processOutput.ErrorLines).Aggregate((x, y) => x + Environment.NewLine + y);
+                    File.WriteAllText(resultsPath, output);
+                }
+
+                Process.Start(resultsPath);
+            }
+
+            return new TestResult(processOutput.ExitCode == 0, assemblyName, span);
+        }
+    }
+}