Merge pull request #9405 from jaredpar/cache

Improve web caching data

src/EditorFeatures/Test2/Diagnostics/GenerateFromUsage/GenerateMethodCrossLanguageTests.vb

@@ -20,7 +20,8 @@ Namespace Microsoft.CodeAnalysis.Editor.UnitTests.Diagnostics.GenerateMethod
             End If
         End Function
 
-        <Fact, Trait(Traits.Feature, Traits.Features.CodeActionsGenerateMethod)>
+        <Fact(Skip:="https://github.com/dotnet/roslyn/issues/9412")>
+        <Trait(Traits.Feature, Traits.Features.CodeActionsGenerateMethod)>
         Public Async Function TestSimpleInstanceMethod_CSharpToVisualBasic() As System.Threading.Tasks.Task
             Dim input =
         <Workspace>

src/EditorFeatures/Test2/Rename/RenameEngineTests.vb

@@ -3128,7 +3128,8 @@ class Program
         End Sub
 
         <WorkItem(622086, "http://vstfdevdiv:8080/DevDiv2/DevDiv/_workitems/edit/622086")>
-        <Fact, Trait(Traits.Feature, Traits.Features.Rename)>
+        <Fact(Skip:="https://github.com/dotnet/roslyn/issues/9412")>
+        <Trait(Traits.Feature, Traits.Features.Rename)>
         Public Sub Bug622086_CSRenameExplicitInterfaceImplementation()
             Using result = RenameEngineResult.Create(
                 <Workspace>

src/Tools/Source/RunTests/Cache/CachingTestExecutor.cs

@@ -14,6 +14,8 @@ internal sealed class CachingTestExecutor : ITestExecutor
         private readonly ContentUtil _contentUtil;
         private readonly IDataStorage _dataStorage;
 
+        public IDataStorage DataStorage => _dataStorage;
+
         internal CachingTestExecutor(Options options, ITestExecutor testExecutor, IDataStorage dataStorage)
         {
             _testExecutor = testExecutor;
@@ -76,7 +78,8 @@ private TestResult Migrate(string assemblyPath, CachedTestResult cachedTestResul
                 commandLine: commandLine,
                 elapsed: TimeSpan.FromMilliseconds(0),
                 standardOutput: cachedTestResult.StandardOutput,
-                errorOutput: cachedTestResult.ErrorOutput);
+                errorOutput: cachedTestResult.ErrorOutput,
+                isResultFromCache: true);
         }
 
         private async Task CacheTestResult(ContentFile contentFile, TestResult testResult)
@@ -91,7 +94,7 @@ private async Task CacheTestResult(ContentFile contentFile, TestResult testResul
                     resultsFileName: Path.GetFileName(testResult.ResultsFilePath),
                     resultsFileContent: resultFileContent,
                     ellapsed: testResult.Elapsed);
-                await _dataStorage.AddCachedTestResult(contentFile, cachedTestResult).ConfigureAwait(true);
+                await _dataStorage.AddCachedTestResult(testResult.AssemblyName, contentFile, cachedTestResult).ConfigureAwait(true);
             }
             catch (Exception ex)
             {

src/Tools/Source/RunTests/Cache/EmptyDataStorage.cs

+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace RunTests.Cache
+{
+    internal sealed class EmptyDataStorage : IDataStorage
+    {
+        internal static readonly EmptyDataStorage Instance = new EmptyDataStorage();
+
+        public string Name => "none";
+
+        public Task AddCachedTestResult(string assemblyName, ContentFile conentFile, CachedTestResult testResult)
+        {
+            var source = new TaskCompletionSource<bool>();
+            source.SetResult(true);
+            return source.Task;
+        }
+
+        public Task<CachedTestResult?> TryGetCachedTestResult(string checksum)
+        {
+            return Task.FromResult<CachedTestResult?>(null);
+        }
+    }
+}

src/Tools/Source/RunTests/Cache/IDataStorage.cs

@@ -10,9 +10,11 @@ namespace RunTests.Cache
 {
     internal interface IDataStorage
     {
+        string Name { get; }
+
         Task<CachedTestResult?> TryGetCachedTestResult(string checksum);
 
-        Task AddCachedTestResult(ContentFile conentFile, CachedTestResult testResult);
+        Task AddCachedTestResult(string assemblyName, ContentFile conentFile, CachedTestResult testResult);
     }
 
     internal struct CachedTestResult

src/Tools/Source/RunTests/Cache/LocalDataStorage.cs

@@ -31,6 +31,8 @@ private enum StorageKind
 
         private readonly string _storagePath;
 
+        public string Name => "local";
+
         internal LocalDataStorage(string storagePath = null)
         {
             _storagePath = storagePath ?? Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), DirectoryName);
@@ -85,7 +87,7 @@ public bool TryGetCachedTestResult(string checksum, out CachedTestResult testRes
             return false;
         }
 
-        public Task AddCachedTestResult(ContentFile contentFile, CachedTestResult testResult)
+        public Task AddCachedTestResult(string assemblyName, ContentFile contentFile, CachedTestResult testResult)
         {
             var checksum = contentFile.Checksum;
             var storagePath = Path.Combine(_storagePath, checksum);

src/Tools/Source/RunTests/Cache/WebDataStorage.cs

@@ -20,15 +20,16 @@ internal sealed class WebDataStorage : IDataStorage
         private const string NameResultsFileName = "ResultsFileName";
         private const string NameResultsFileContent = "ResultsFileContent";
         private const string NameEllapsedSeconds = "EllapsedSeconds";
-        private const string DashboardUriString = "http://jdash.azurewebsites.net";
 
-        private readonly RestClient _restClient = new RestClient(DashboardUriString);
+        private readonly RestClient _restClient = new RestClient(Constants.DashboardUriString);
 
-        public async Task AddCachedTestResult(ContentFile contentFile, CachedTestResult testResult)
+        public string Name => "web";
+
+        public async Task AddCachedTestResult(string assemblyName, ContentFile contentFile, CachedTestResult testResult)
         {
             var obj = new JObject();
             obj["TestResultData"] = CreateTestResultData(testResult);
-            obj["TestSourceData"] = CreateTestSourceData();
+            obj["TestSourceData"] = CreateTestSourceData(assemblyName);
 
             var request = new RestRequest($"api/testcache/{contentFile.Checksum}");
             request.Method = Method.PUT;
@@ -72,15 +73,17 @@ private static JObject CreateTestResultData(CachedTestResult testResult)
             obj[NameOutputStandard] = testResult.ErrorOutput;
             obj[NameResultsFileName] = testResult.ResultsFileName;
             obj[NameResultsFileContent] = testResult.ResultsFileContent;
-            obj[NameEllapsedSeconds] = testResult.Ellapsed.TotalSeconds;
+            obj[NameEllapsedSeconds] = (int)testResult.Ellapsed.TotalSeconds;
             return obj;
         }
 
-        private static JObject CreateTestSourceData()
+        private JObject CreateTestSourceData(string assemblyName)
         {
             var obj = new JObject();
             obj["MachineName"] = Environment.MachineName;
             obj["TestRoot"] = "";
+            obj["AssemblyName"] = assemblyName;
+            obj["IsJenkins"] = Constants.IsJenkinsRun;
             return obj;
         }
     }

src/Tools/Source/RunTests/Constants.cs

@@ -10,6 +10,10 @@ namespace RunTests
 {
     internal static class Constants
     {
-        internal const string ResultsDirectoryName = "xUnitResults";
+        internal static string ResultsDirectoryName => "xUnitResults";
+
+        internal static bool IsJenkinsRun => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("JENKINS_URL"));
+
+        internal static string DashboardUriString => "http://jdash.azurewebsites.net";
     }
 }

src/Tools/Source/RunTests/ITestExecutor.cs

 // Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
 
+using RunTests.Cache;
 using System;
 using System.IO;
 using System.Threading;
@@ -16,6 +17,7 @@ internal struct TestResult
         internal TimeSpan Elapsed { get; }
         internal string StandardOutput { get; }
         internal string ErrorOutput { get; }
+        internal bool IsResultFromCache { get; }
 
         /// <summary>
         /// Path to the results file.  Can be null in the case xunit error'd and did not create one. 
@@ -25,7 +27,7 @@ internal struct TestResult
         internal string ResultDir { get; }
         internal bool Succeeded => ExitCode == 0;
 
-        internal TestResult(int exitCode, string assemblyPath, string resultDir, string resultsFilePath, string commandLine, TimeSpan elapsed, string standardOutput, string errorOutput)
+        internal TestResult(int exitCode, string assemblyPath, string resultDir, string resultsFilePath, string commandLine, TimeSpan elapsed, string standardOutput, string errorOutput, bool isResultFromCache)
         {
             ExitCode = exitCode;
             AssemblyName = Path.GetFileName(assemblyPath);
@@ -36,11 +38,14 @@ internal TestResult(int exitCode, string assemblyPath, string resultDir, string
             Elapsed = elapsed;
             StandardOutput = standardOutput;
             ErrorOutput = errorOutput;
+            IsResultFromCache = isResultFromCache;
         }
     }
 
     internal interface ITestExecutor
     {
+        IDataStorage DataStorage { get; }
+
         string GetCommandLine(string assemblyPath);
 
         Task<TestResult> RunTestAsync(string assemblyPath, CancellationToken cancellationToken);

src/Tools/Source/RunTests/ProcessTestExecutor.cs

 // Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
 
+using RunTests.Cache;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
@@ -15,6 +16,8 @@ internal sealed class ProcessTestExecutor : ITestExecutor
     {
         private readonly Options _options;
 
+        public IDataStorage DataStorage => EmptyDataStorage.Instance;
+
         internal ProcessTestExecutor(Options options)
         {
             _options = options;
@@ -125,7 +128,8 @@ public async Task<TestResult> RunTestAsync(string assemblyPath, CancellationToke
                     commandLine: commandLine,
                     elapsed: span,
                     standardOutput: standardOutput,
-                    errorOutput: errorOutput);
+                    errorOutput: errorOutput,
+                    isResultFromCache: false);
             }
             catch (Exception ex)
             {

src/Tools/Source/RunTests/Program.cs

@@ -9,6 +9,8 @@
 using System.Threading;
 using System.Threading.Tasks;
 using RunTests.Cache;
+using Newtonsoft.Json.Linq;
+using RestSharp;
 
 namespace RunTests
 {
@@ -30,31 +32,21 @@ internal static int Main(string[] args)
                 cts.Cancel();
             };
 
-            ITestExecutor testExecutor = new ProcessTestExecutor(options);
-            if (options.UseCachedResults)
-            {
-                // The web caching layer is still being worked on.  For now want to limit it to Roslyn developers
-                // and Jenkins runs by default until we work on this a bit more.  Anyone reading this who wants
-                // to try it out should feel free to opt into this. 
-                IDataStorage dataStorage = new LocalDataStorage();
-                if (StringComparer.OrdinalIgnoreCase.Equals("REDMOND", Environment.UserDomainName) || 
-                    !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("JENKINS_URL")))
-                {
-                    Console.WriteLine("Using web cache");
-                    dataStorage = new WebDataStorage();
-                }
-
-                testExecutor = new CachingTestExecutor(options, testExecutor, dataStorage);
+            return RunCore(options, cts.Token).GetAwaiter().GetResult();
         }
 
+        private static async Task<int> RunCore(Options options, CancellationToken cancellationToken)
+        {
+            var testExecutor = CreateTestExecutor(options);
             var testRunner = new TestRunner(options, testExecutor);
             var start = DateTime.Now;
 
-            Console.WriteLine("Running {0} test assemblies", options.Assemblies.Count());
+            Console.WriteLine($"Data Storage: {testExecutor.DataStorage.Name}");
+            Console.WriteLine($"Running {options.Assemblies.Count()} test assemblies");
 
             var orderedList = OrderAssemblyList(options.Assemblies);
-            var result = testRunner.RunAllAsync(orderedList, cts.Token).Result;
-            var span = DateTime.Now - start;
+            var result = await testRunner.RunAllAsync(orderedList, cancellationToken).ConfigureAwait(true);
+            var ellapsed = DateTime.Now - start;
 
             foreach (var assemblyPath in options.MissingAssemblies)
             {
@@ -63,22 +55,89 @@ internal static int Main(string[] args)
 
             Logger.Finish();
 
-            if (!result)
+            if (CanUseWebStorage())
             {
-                ConsoleUtil.WriteLine(ConsoleColor.Red, "Test failures encountered: {0}", span);
+                await SendRunStats(options, testExecutor.DataStorage, ellapsed, result, cancellationToken).ConfigureAwait(true);
+            }
+
+            if (!result.Succeeded)
+            {
+                ConsoleUtil.WriteLine(ConsoleColor.Red, $"Test failures encountered: {ellapsed}");
                 return 1;
             }
 
-            Console.WriteLine("All tests passed: {0}", span);
+            Console.WriteLine($"All tests passed: {ellapsed}");
             return options.MissingAssemblies.Any() ? 1 : 0;
         }
 
+        private static bool CanUseWebStorage()
+        {
+            // The web caching layer is still being worked on.  For now want to limit it to Roslyn developers
+            // and Jenkins runs by default until we work on this a bit more.  Anyone reading this who wants
+            // to try it out should feel free to opt into this. 
+            return 
+                StringComparer.OrdinalIgnoreCase.Equals("REDMOND", Environment.UserDomainName) || 
+                Constants.IsJenkinsRun;
+        }
+
+        private static ITestExecutor CreateTestExecutor(Options options)
+        {
+            var processTestExecutor = new ProcessTestExecutor(options);
+            if (!options.UseCachedResults)
+            {
+                return processTestExecutor;
+            }
+
+            // The web caching layer is still being worked on.  For now want to limit it to Roslyn developers
+            // and Jenkins runs by default until we work on this a bit more.  Anyone reading this who wants
+            // to try it out should feel free to opt into this. 
+            IDataStorage dataStorage = new LocalDataStorage();
+            if (CanUseWebStorage())
+            {
+                dataStorage = new WebDataStorage();
+            }
+
+            return new CachingTestExecutor(options, processTestExecutor, dataStorage);
+        }
+
         /// <summary>
         /// Order the assembly list so that the largest assemblies come first.  This
         /// is not ideal as the largest assembly does not necessarily take the most time.
         /// </summary>
         /// <param name="list"></param>
-        private static IOrderedEnumerable<string> OrderAssemblyList(IEnumerable<string> list) =>
-            list.OrderByDescending((assemblyName) => new FileInfo(assemblyName).Length);
+        private static IOrderedEnumerable<string> OrderAssemblyList(IEnumerable<string> list)
+        {
+            return list.OrderByDescending((assemblyName) => new FileInfo(assemblyName).Length);
+        }
+
+        private static async Task SendRunStats(Options options, IDataStorage dataStorage, TimeSpan ellapsed, RunAllResult result, CancellationToken cancellationToken)
+        {
+            var obj = new JObject();
+            obj["Cache"] = dataStorage.Name;
+            obj["EllapsedSeconds"] = (int)ellapsed.TotalSeconds;
+            obj["IsJenkins"] = Constants.IsJenkinsRun;
+            obj["Is32Bit"] = !options.Test64;
+            obj["AssemblyCount"] = options.Assemblies.Count;
+            obj["CacheCount"] = result.CacheCount;
+            obj["Succeeded"] = result.Succeeded;
+
+            var request = new RestRequest("api/testrun", Method.POST);
+            request.RequestFormat = DataFormat.Json;
+            request.AddParameter("text/json", obj.ToString(), ParameterType.RequestBody);
+
+            try
+            {
+                var client = new RestClient(Constants.DashboardUriString);
+                var response = await client.ExecuteTaskAsync(request);
+                if (response.StatusCode != System.Net.HttpStatusCode.NoContent)
+                {
+                    Logger.Log($"Unable to send results: {response.ErrorMessage}");
+                }
+            }
+            catch
+            {
+                Logger.Log("Unable to send results");
+            }
+        }
     }
 }
\ No newline at end of file

src/Tools/Source/RunTests/RunTests.csproj

@@ -30,6 +30,7 @@
     <Compile Include="Cache\ContentUtil.cs" />
     <Compile Include="Cache\CachingTestExecutor.cs" />
     <Compile Include="Cache\ConentFile.cs" />
+    <Compile Include="Cache\EmptyDataStorage.cs" />
     <Compile Include="Cache\WebDataStorage.cs" />
     <Compile Include="Constants.cs" />
     <Compile Include="FileUtil.cs" />

src/Tools/Source/RunTests/TestRunner.cs

@@ -12,6 +12,18 @@
 
 namespace RunTests
 {
+    internal struct RunAllResult
+    {
+        internal readonly bool Succeeded;
+        internal readonly int CacheCount;
+
+        internal RunAllResult(bool succeeded, int cacheCount)
+        {
+            Succeeded = succeeded;
+            CacheCount = cacheCount;
+        }
+    }
+
     internal sealed class TestRunner
     {
         private readonly ITestExecutor _testExecutor;
@@ -23,10 +35,11 @@ internal TestRunner(Options options, ITestExecutor testExecutor)
             _options = options;
         }
 
-        internal async Task<bool> RunAllAsync(IEnumerable<string> assemblyList, CancellationToken cancellationToken)
+        internal async Task<RunAllResult> RunAllAsync(IEnumerable<string> assemblyList, CancellationToken cancellationToken)
         {
             var max = (int)Environment.ProcessorCount * 1.5;
             var allPassed = true;
+            var cacheCount = 0;
             var waiting = new Stack<string>(assemblyList);
             var running = new List<Task<TestResult>>();
             var completed = new List<TestResult>();
@@ -49,6 +62,11 @@ internal async Task<bool> RunAllAsync(IEnumerable<string> assemblyList, Cancella
                                 allPassed = false;
                             }
 
+                            if (testResult.IsResultFromCache)
+                            {
+                                cacheCount++;
+                            }
+
                             completed.Add(testResult);
                         }
                         catch (Exception ex)
@@ -77,7 +95,7 @@ internal async Task<bool> RunAllAsync(IEnumerable<string> assemblyList, Cancella
 
             Print(completed);
 
-            return allPassed;
+            return new RunAllResult(allPassed, cacheCount);
         }
 
         private void Print(List<TestResult> testResults)