Chunk up assembly support

src/Tools/Source/RunTests/AssemblyScheduler.cs

+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using System.Reflection.Metadata;
+using System.Reflection.PortableExecutable;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace RunTests
+{
+    internal struct AssemblyInfo
+    {
+        internal readonly string AssemblyPath;
+        internal readonly string DisplayName;
+        internal readonly string ResultsFileName;
+        internal readonly string ExtraArguments;
+
+        internal AssemblyInfo(
+            string assemblyPath,
+            string displayName,
+            string resultsFileName,
+            string extraArguments)
+        {
+            AssemblyPath = assemblyPath;
+            DisplayName = displayName;
+            ResultsFileName = resultsFileName;
+            ExtraArguments = extraArguments;
+        }
+
+        public override string ToString() => DisplayName;
+    }
+
+    internal sealed class AssemblyScheduler
+    {
+        private readonly StringBuilder _builder;
+        private readonly Options _options;
+
+        internal AssemblyScheduler(Options options)
+        {
+            _builder = new StringBuilder();
+            _options = options;
+        }
+
+        internal IEnumerable<AssemblyInfo> Schedule(IEnumerable<string> assemblyPaths)
+        {
+            var list = new List<AssemblyInfo>();
+            foreach (var assemblyPath in assemblyPaths)
+            {
+                list.AddRange(Schedule(assemblyPath));
+            }
+
+            return list;
+        }
+
+        private IEnumerable<AssemblyInfo> Schedule(string assemblyPath)
+        {
+            var scheduleList = new List<AssemblyInfo>();
+            var id = 0;
+            var count = 0;
+            foreach (var tuple in GetClassNames(assemblyPath))
+            {
+                count += tuple.Item2;
+                _builder.Append($@"-class ""{tuple.Item1}"" ");
+                if (count > 750)
+                {
+                    scheduleList.Add(CreateAssemblyInfo(assemblyPath, id, _builder.ToString()));
+                    _builder.Length = 0;
+                    count = 0;
+                    id++;
+                }
+            }
+
+            if (_builder.Length > 0)
+            {
+                scheduleList.Add(CreateAssemblyInfo(assemblyPath, id, _builder.ToString()));
+                _builder.Length = 0;
+            }
+
+            return scheduleList;
+        }
+
+        private AssemblyInfo CreateAssemblyInfo(string assemblyPath, int id, string arguments)
+        {
+            var assemblyName = Path.GetFileName(assemblyPath);
+            var displayName = $"{assemblyName}.{id}";
+            var suffix = _options.UseHtml ? "html" : "xml";
+            var resultsFileName = $"{assemblyName}.{id}.{suffix}";
+            return new AssemblyInfo(
+                assemblyPath,
+                displayName,
+                resultsFileName,
+                arguments);
+        }
+
+        private List<Tuple<string, int>> GetClassNames(string assemblyPath)
+        { 
+            using (var stream = File.OpenRead(assemblyPath))
+            using (var peReader = new PEReader(stream))
+            {
+                var metadataReader = peReader.GetMetadataReader();
+                return GetClassNames(metadataReader);
+            }
+        }
+
+        private List<Tuple<string, int>> GetClassNames(MetadataReader reader)
+        {
+            var list = new List<Tuple<string, int>>();
+            foreach (var handle in reader.TypeDefinitions)
+            {
+                var type = reader.GetTypeDefinition(handle);
+                var methodCount = GetMethodCount(reader, type);
+                if (methodCount == 0)
+                {
+                    continue;
+                }
+
+                var namespaceName = reader.GetString(type.Namespace);
+                var typeName = reader.GetString(type.Name);
+                var fullName = $"{namespaceName}.{typeName}";
+                list.Add(Tuple.Create(fullName, methodCount));
+            }
+
+            return list;
+        }
+
+        private int GetMethodCount(MetadataReader reader, TypeDefinition type)
+        {
+            if (TypeAttributes.Public != (type.Attributes & TypeAttributes.Public))
+            {
+                return 0;
+            }
+
+            var count = 0;
+            foreach (var handle in type.GetMethods())
+            {
+                var methodDefinition = reader.GetMethodDefinition(handle);
+                if (methodDefinition.GetCustomAttributes().Count == 0)
+                {
+                    continue;
+                }
+
+                if (MethodAttributes.Public != (methodDefinition.Attributes & MethodAttributes.Public))
+                {
+                    continue;
+                }
+
+                count++;
+            }
+
+            return count;
+        }
+    }
+}

src/Tools/Source/RunTests/Cache/CachingTestExecutor.cs

@@ -23,13 +23,15 @@ internal CachingTestExecutor(Options options, ITestExecutor testExecutor, IDataS
             _contentUtil = new ContentUtil(options);
         }
 
-        public string GetCommandLine(string assemblyPath)
+        public string GetCommandLine(AssemblyInfo assemblyInfo)
         {
-            return _testExecutor.GetCommandLine(assemblyPath);
+            return _testExecutor.GetCommandLine(assemblyInfo);
         }
 
-        public async Task<TestResult> RunTestAsync(string assemblyPath, CancellationToken cancellationToken)
+        public async Task<TestResult> RunTestAsync(AssemblyInfo assemblyInfo, CancellationToken cancellationToken)
         {
+            throw new Exception();
+            /*
             var contentFile = _contentUtil.GetTestResultContentFile(assemblyPath);
             var builder = new StringBuilder();
             builder.AppendLine($"{Path.GetFileName(assemblyPath)} - {contentFile.Checksum}");
@@ -56,6 +58,7 @@ public async Task<TestResult> RunTestAsync(string assemblyPath, CancellationToke
             var testResult = await _testExecutor.RunTestAsync(assemblyPath, cancellationToken);
             await CacheTestResult(contentFile, testResult).ConfigureAwait(true);
             return testResult;
+            */
         }
 
         /// <summary>
@@ -64,6 +67,8 @@ public async Task<TestResult> RunTestAsync(string assemblyPath, CancellationToke
         /// </summary>
         private TestResult Migrate(string assemblyPath, CachedTestResult cachedTestResult)
         {
+            throw new Exception();
+            /*
             var resultsDir = Path.Combine(Path.GetDirectoryName(assemblyPath), Constants.ResultsDirectoryName);
             FileUtil.EnsureDirectory(resultsDir);
             var resultsFilePath = Path.Combine(resultsDir, cachedTestResult.ResultsFileName);
@@ -80,6 +85,7 @@ private TestResult Migrate(string assemblyPath, CachedTestResult cachedTestResul
                 standardOutput: cachedTestResult.StandardOutput,
                 errorOutput: cachedTestResult.ErrorOutput,
                 isResultFromCache: true);
+                */
         }
 
         private async Task CacheTestResult(ContentFile contentFile, TestResult testResult)

src/Tools/Source/RunTests/ITestExecutor.cs

@@ -46,8 +46,8 @@ internal interface ITestExecutor
     {
         IDataStorage DataStorage { get; }
 
-        string GetCommandLine(string assemblyPath);
+        string GetCommandLine(AssemblyInfo assemblyInfo);
 
-        Task<TestResult> RunTestAsync(string assemblyPath, CancellationToken cancellationToken);
+        Task<TestResult> RunTestAsync(AssemblyInfo assemblyInfo, CancellationToken cancellationToken);
     }
 }

src/Tools/Source/RunTests/ProcessTestExecutor.cs

@@ -23,18 +23,19 @@ internal ProcessTestExecutor(Options options)
             _options = options;
         }
 
-        public string GetCommandLine(string assemblyPath)
+        public string GetCommandLine(AssemblyInfo assemblyInfo)
         {
-            return $"{_options.XunitPath} {GetCommandLineArguments(assemblyPath)}";
+            return $"{_options.XunitPath} {GetCommandLineArguments(assemblyInfo)}";
         }
 
-        public string GetCommandLineArguments(string assemblyPath)
+        public string GetCommandLineArguments(AssemblyInfo assemblyInfo)
         {
-            var assemblyName = Path.GetFileName(assemblyPath);
-            var resultsFilePath = GetResultsFilePath(assemblyPath);
+            var assemblyName = Path.GetFileName(assemblyInfo.AssemblyPath);
+            var resultsFilePath = GetResultsFilePath(assemblyInfo);
 
             var builder = new StringBuilder();
-            builder.AppendFormat(@"""{0}""", assemblyPath);
+            builder.AppendFormat(@"""{0}""", assemblyInfo.AssemblyPath);
+            builder.AppendFormat(@" {0}", assemblyInfo.ExtraArguments);
             builder.AppendFormat(@" -{0} ""{1}""", _options.UseHtml ? "html" : "xml", resultsFilePath);
             builder.Append(" -noshadow -verbose");
 
@@ -59,19 +60,18 @@ public string GetCommandLineArguments(string assemblyPath)
             return builder.ToString();
         }
 
-        private string GetResultsFilePath(string assemblyPath)
+        private string GetResultsFilePath(AssemblyInfo assemblyInfo)
         {
-            var assemblyName = Path.GetFileName(assemblyPath);
-            var resultsDir = Path.Combine(Path.GetDirectoryName(assemblyPath), Constants.ResultsDirectoryName);
-            return Path.Combine(resultsDir, $"{assemblyName}.{(_options.UseHtml ? "html" : "xml")}");
+            var resultsDir = Path.Combine(Path.GetDirectoryName(assemblyInfo.AssemblyPath), Constants.ResultsDirectoryName);
+            return Path.Combine(resultsDir, assemblyInfo.ResultsFileName);
         }
 
-        public async Task<TestResult> RunTestAsync(string assemblyPath, CancellationToken cancellationToken)
+        public async Task<TestResult> RunTestAsync(AssemblyInfo assemblyInfo, CancellationToken cancellationToken)
         {
             try
             {
-                var commandLineArguments = GetCommandLineArguments(assemblyPath);
-                var resultsFilePath = GetResultsFilePath(assemblyPath);
+                var commandLineArguments = GetCommandLineArguments(assemblyInfo);
+                var resultsFilePath = GetResultsFilePath(assemblyInfo);
                 var resultsDir = Path.GetDirectoryName(resultsFilePath);
 
                 // NOTE: xUnit doesn't always create the log directory
@@ -119,13 +119,14 @@ public async Task<TestResult> RunTestAsync(string assemblyPath, CancellationToke
                     }
                 }
 
-                var commandLine = GetCommandLine(assemblyPath);
+                var commandLine = GetCommandLine(assemblyInfo);
                 var standardOutput = string.Join(Environment.NewLine, processOutput.OutputLines);
                 var errorOutput = string.Join(Environment.NewLine, processOutput.ErrorLines);
 
+                // TODO: Need to include assemblyinfo here? 
                 return new TestResult(
                     exitCode: processOutput.ExitCode,
-                    assemblyPath: assemblyPath,
+                    assemblyPath: assemblyInfo.AssemblyPath,
                     resultDir: resultsDir,
                     resultsFilePath: resultsFilePath,
                     commandLine: commandLine,
@@ -136,7 +137,7 @@ public async Task<TestResult> RunTestAsync(string assemblyPath, CancellationToke
             }
             catch (Exception ex)
             {
-                throw new Exception($"Unable to run {assemblyPath} with {_options.XunitPath}. {ex}");
+                throw new Exception($"Unable to run {assemblyInfo.AssemblyPath} with {_options.XunitPath}. {ex}");
             }
         }
     }

src/Tools/Source/RunTests/Program.cs

@@ -37,6 +37,16 @@ internal static int Main(string[] args)
 
         private static async Task<int> RunCore(Options options, CancellationToken cancellationToken)
         {
+            if (options.MissingAssemblies.Count > 0)
+            {
+                foreach (var assemblyPath in options.MissingAssemblies)
+                {
+                    ConsoleUtil.WriteLine(ConsoleColor.Red, $"The file '{assemblyPath}' does not exist, is an invalid file name, or you do not have sufficient permissions to read the specified file.");
+                }
+
+                return 1;
+            }
+
             var testExecutor = CreateTestExecutor(options);
             var testRunner = new TestRunner(options, testExecutor);
             var start = DateTime.Now;
@@ -44,15 +54,13 @@ private static async Task<int> RunCore(Options options, CancellationToken cancel
             Console.WriteLine($"Data Storage: {testExecutor.DataStorage.Name}");
             Console.WriteLine($"Running {options.Assemblies.Count()} test assemblies");
 
-            var orderedList = OrderAssemblyList(options.Assemblies);
-            var result = await testRunner.RunAllAsync(orderedList, cancellationToken).ConfigureAwait(true);
+            // TODO: Do we still need to order by file size? 
+            // var orderedList = OrderAssemblyList(options.Assemblies);
+            var scheduler = new AssemblyScheduler(options);
+            var assemblyInfoList = scheduler.Schedule(options.Assemblies);
+            var result = await testRunner.RunAllAsync(assemblyInfoList, cancellationToken).ConfigureAwait(true);
             var ellapsed = DateTime.Now - start;
 
-            foreach (var assemblyPath in options.MissingAssemblies)
-            {
-                ConsoleUtil.WriteLine(ConsoleColor.Red, $"The file '{assemblyPath}' does not exist, is an invalid file name, or you do not have sufficient permissions to read the specified file.");
-            }
-
             Logger.Finish();
 
             if (CanUseWebStorage())
@@ -88,6 +96,10 @@ private static ITestExecutor CreateTestExecutor(Options options)
                 return processTestExecutor;
             }
 
+            return processTestExecutor;
+
+            /* TODO: fix this
+
             // The web caching layer is still being worked on.  For now want to limit it to Roslyn developers
             // and Jenkins runs by default until we work on this a bit more.  Anyone reading this who wants
             // to try it out should feel free to opt into this. 
@@ -98,6 +110,7 @@ private static ITestExecutor CreateTestExecutor(Options options)
             }
 
             return new CachingTestExecutor(options, processTestExecutor, dataStorage);
+            */
         }
 
         /// <summary>

src/Tools/Source/RunTests/RunTests.csproj

@@ -27,6 +27,7 @@
     <Reference Include="System.Xml" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="AssemblyScheduler.cs" />
     <Compile Include="Cache\ContentUtil.cs" />
     <Compile Include="Cache\CachingTestExecutor.cs" />
     <Compile Include="Cache\ConentFile.cs" />

src/Tools/Source/RunTests/TestRunner.cs

@@ -35,12 +35,12 @@ internal TestRunner(Options options, ITestExecutor testExecutor)
             _options = options;
         }
 
-        internal async Task<RunAllResult> RunAllAsync(IEnumerable<string> assemblyList, CancellationToken cancellationToken)
+        internal async Task<RunAllResult> RunAllAsync(IEnumerable<AssemblyInfo> assemblyInfoList, CancellationToken cancellationToken)
         {
             var max = (int)Environment.ProcessorCount * 1.5;
             var allPassed = true;
             var cacheCount = 0;
-            var waiting = new Stack<string>(assemblyList);
+            var waiting = new Stack<AssemblyInfo>(assemblyInfoList);
             var running = new List<Task<TestResult>>();
             var completed = new List<TestResult>();
 

src/Tools/Source/RunTests/project.json

@@ -2,6 +2,8 @@
   "dependencies": { 
     "Newtonsoft.Json": "8.0.2",
     "RestSharp": "105.2.3",
+    "System.Collections.Immutable": "1.1.37",
+    "System.Reflection.Metadata": "1.2.0-rc2-23826"
   },
   "frameworks": {
     "net45": { }