Merge pull request #9312 from jaredpar/cache

Use web storage for test caching layer

BuildAndTest.proj

@@ -96,7 +96,7 @@
 
     <Exec Command="$(CoreClrTestDirectory)\CoreRun.exe $(CoreClrTestDirectory)\xunit.console.netcore.exe @(CoreTestAssemblies, ' ') -parallel all -xml $(CoreClrTestDirectory)\xUnitResults\TestResults.xml" />
 
-    <Exec Command="Binaries\$(Configuration)\RunTests.exe $(NuGetPackageRoot)\xunit.runner.console\$(XunitVersion)\tools $(RunTestArgs) @(TestAssemblies, ' ')" />
+    <Exec Command="Binaries\$(Configuration)\RunTests\RunTests.exe $(NuGetPackageRoot)\xunit.runner.console\$(XunitVersion)\tools $(RunTestArgs) @(TestAssemblies, ' ')" />
 
   </Target>
 

src/Tools/Source/RunTests/Cache/CachingTestExecutor.cs

@@ -36,21 +36,23 @@ public async Task<TestResult> RunTestAsync(string assemblyPath, CancellationToke
             builder.AppendLine("===");
             Logger.Log(builder.ToString());
 
-            TestResult testResult;
-            CachedTestResult cachedTestResult;
-            if (!_dataStorage.TryGetCachedTestResult(contentFile.Checksum, out cachedTestResult))
+            try
             {
-                Logger.Log($"{Path.GetFileName(assemblyPath)} - running");
-                testResult = await _testExecutor.RunTestAsync(assemblyPath, cancellationToken);
-                Logger.Log($"{Path.GetFileName(assemblyPath)} - caching");
-                CacheTestResult(contentFile, testResult);
+                var cachedTestResult = await _dataStorage.TryGetCachedTestResult(contentFile.Checksum);
+                if (cachedTestResult.HasValue)
+                {
+                    Logger.Log($"{Path.GetFileName(assemblyPath)} - cache hit");
+                    return Migrate(assemblyPath, cachedTestResult.Value);
+                }
             }
-            else
+            catch (Exception ex)
             {
-                testResult = Migrate(assemblyPath, cachedTestResult);
-                Logger.Log($"{Path.GetFileName(assemblyPath)} - cache hit");
+                Logger.Log($"Error reading cache {ex}");
             }
 
+            Logger.Log($"{Path.GetFileName(assemblyPath)} - running");
+            var testResult = await _testExecutor.RunTestAsync(assemblyPath, cancellationToken);
+            await CacheTestResult(contentFile, testResult).ConfigureAwait(true);
             return testResult;
         }
 
@@ -77,7 +79,7 @@ private TestResult Migrate(string assemblyPath, CachedTestResult cachedTestResul
                 errorOutput: cachedTestResult.ErrorOutput);
         }
 
-        private void CacheTestResult(ContentFile contentFile, TestResult testResult)
+        private async Task CacheTestResult(ContentFile contentFile, TestResult testResult)
         {
             try
             {
@@ -87,8 +89,9 @@ private void CacheTestResult(ContentFile contentFile, TestResult testResult)
                     standardOutput: testResult.StandardOutput,
                     errorOutput: testResult.ErrorOutput,
                     resultsFileName: Path.GetFileName(testResult.ResultsFilePath),
-                    resultsFileContent: resultFileContent);
-                _dataStorage.AddCachedTestResult(contentFile, cachedTestResult);
+                    resultsFileContent: resultFileContent,
+                    ellapsed: testResult.Elapsed);
+                await _dataStorage.AddCachedTestResult(contentFile, cachedTestResult).ConfigureAwait(true);
             }
             catch (Exception ex)
             {

src/Tools/Source/RunTests/Cache/IDataStorage.cs

@@ -10,9 +10,9 @@ namespace RunTests.Cache
 {
     internal interface IDataStorage
     {
-        bool TryGetCachedTestResult(string checksum, out CachedTestResult testResult);
+        Task<CachedTestResult?> TryGetCachedTestResult(string checksum);
 
-        void AddCachedTestResult(ContentFile conentFile, CachedTestResult testResult);
+        Task AddCachedTestResult(ContentFile conentFile, CachedTestResult testResult);
     }
 
     internal struct CachedTestResult
@@ -22,19 +22,22 @@ internal struct CachedTestResult
         internal string ErrorOutput { get; }
         internal string ResultsFileName { get; }
         internal string ResultsFileContent { get; }
+        internal TimeSpan Ellapsed { get; }
 
         internal CachedTestResult(
             int exitCode,
             string standardOutput,
             string errorOutput,
             string resultsFileName,
-            string resultsFileContent)
+            string resultsFileContent,
+            TimeSpan ellapsed)
         {
             ExitCode = exitCode;
             StandardOutput = standardOutput;
             ErrorOutput = errorOutput;
             ResultsFileName = resultsFileName;
             ResultsFileContent = resultsFileContent;
+            Ellapsed = ellapsed;
         }
     }
 }

src/Tools/Source/RunTests/Cache/LocalDataStorage.cs

@@ -22,6 +22,7 @@ private enum StorageKind
             ErrorOutput,
             ResultsFileContent,
             ResultsFileName,
+            EllapsedSeconds,
             Content
         }
 
@@ -35,8 +36,20 @@ internal LocalDataStorage(string storagePath = null)
             _storagePath = storagePath ?? Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), DirectoryName);
         }
 
-        public bool TryGetCachedTestResult(string checksum, out CachedTestResult testResult)
+        public Task<CachedTestResult?> TryGetCachedTestResult(string checksum)
         {
+            CachedTestResult testResult;
+            CachedTestResult? value = null;
+            if (TryGetCachedTestResult(checksum, out testResult))
+            {
+                value = testResult;
+            }
+
+            return Task.FromResult(value);
+        }
+
+        public bool TryGetCachedTestResult(string checksum, out CachedTestResult testResult)
+        { 
             testResult = default(CachedTestResult);
 
             var storageFolder = GetStorageFolder(checksum);
@@ -52,13 +65,15 @@ public bool TryGetCachedTestResult(string checksum, out CachedTestResult testRes
                 var errorOutput = Read(checksum, StorageKind.ErrorOutput);
                 var resultsFileName = Read(checksum, StorageKind.ResultsFileName);
                 var resultsFileContent = Read(checksum, StorageKind.ResultsFileContent);
+                var ellapsed = Read(checksum, StorageKind.EllapsedSeconds);
 
                 testResult = new CachedTestResult(
                     exitCode: int.Parse(exitCode),
                     standardOutput: standardOutput,
                     errorOutput: errorOutput,
                     resultsFileName: resultsFileName,
-                    resultsFileContent: resultsFileContent);
+                    resultsFileContent: resultsFileContent,
+                    ellapsed: TimeSpan.FromSeconds(int.Parse(ellapsed)));
                 return true;
             }
             catch (Exception e)
@@ -70,7 +85,7 @@ public bool TryGetCachedTestResult(string checksum, out CachedTestResult testRes
             return false;
         }
 
-        public void AddCachedTestResult(ContentFile contentFile, CachedTestResult testResult)
+        public Task AddCachedTestResult(ContentFile contentFile, CachedTestResult testResult)
         {
             var checksum = contentFile.Checksum;
             var storagePath = Path.Combine(_storagePath, checksum);
@@ -78,7 +93,7 @@ public void AddCachedTestResult(ContentFile contentFile, CachedTestResult testRe
             {
                 if (!FileUtil.EnsureDirectory(storagePath))
                 {
-                    return;
+                    return Task.FromResult(true);
                 }
 
                 Write(checksum, StorageKind.ExitCode, testResult.ExitCode.ToString());
@@ -86,6 +101,7 @@ public void AddCachedTestResult(ContentFile contentFile, CachedTestResult testRe
                 Write(checksum, StorageKind.ErrorOutput, testResult.ErrorOutput);
                 Write(checksum, StorageKind.ResultsFileName, testResult.ResultsFileName);
                 Write(checksum, StorageKind.ResultsFileContent, testResult.ResultsFileContent);
+                Write(checksum, StorageKind.EllapsedSeconds, testResult.Ellapsed.TotalSeconds.ToString());
                 Write(checksum, StorageKind.Content, contentFile.Content);
             }
             catch (Exception e)
@@ -94,6 +110,8 @@ public void AddCachedTestResult(ContentFile contentFile, CachedTestResult testRe
                 Logger.Log($"Failed to log {checksum} {e.Message}");
                 FileUtil.DeleteDirectory(storagePath);
             }
+
+            return Task.FromResult(true);
         }
 
         private string GetStorageFolder(string checksum)

src/Tools/Source/RunTests/Cache/WebDataStorage.cs

+// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+
+using Newtonsoft.Json.Linq;
+using RestSharp;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace RunTests.Cache
+{
+    internal sealed class WebDataStorage : IDataStorage
+    {
+        private const string NameExitCode = "ExitCode";
+        private const string NameOutputStandard = "OutputStandard";
+        private const string NameOutputError = "OutputError";
+        private const string NameResultsFileName = "ResultsFileName";
+        private const string NameResultsFileContent = "ResultsFileContent";
+        private const string NameEllapsedSeconds = "EllapsedSeconds";
+        private const string DashboardUriString = "http://jdash.azurewebsites.net";
+
+        private readonly RestClient _restClient = new RestClient(DashboardUriString);
+
+        public async Task AddCachedTestResult(ContentFile contentFile, CachedTestResult testResult)
+        {
+            var obj = new JObject();
+            obj["TestResultData"] = CreateTestResultData(testResult);
+            obj["TestSourceData"] = CreateTestSourceData();
+
+            var request = new RestRequest($"api/testcache/{contentFile.Checksum}");
+            request.Method = Method.PUT;
+            request.RequestFormat = DataFormat.Json;
+            request.AddParameter("text/json", obj.ToString(), ParameterType.RequestBody);
+            var response = await _restClient.ExecuteTaskAsync(request);
+        }
+
+        public async Task<CachedTestResult?> TryGetCachedTestResult(string checksum)
+        {
+            try
+            {
+                var request = new RestRequest($"api/testcache/{checksum}");
+                var response = await _restClient.ExecuteGetTaskAsync(request);
+                if (response.StatusCode != HttpStatusCode.OK)
+                {
+                    return null;
+                }
+
+                var obj = JObject.Parse(response.Content);
+                var result = new CachedTestResult(
+                    exitCode: obj.Value<int>(NameExitCode),
+                    standardOutput: obj.Value<string>(NameOutputStandard),
+                    errorOutput: obj.Value<string>(NameOutputError),
+                    resultsFileName: obj.Value<string>(NameResultsFileName),
+                    resultsFileContent: obj.Value<string>(NameResultsFileContent),
+                    ellapsed: TimeSpan.FromSeconds(0));
+                return result;
+            }
+            catch
+            {
+                return null;
+            }
+        }
+
+        private static JObject CreateTestResultData(CachedTestResult testResult)
+        {
+            var obj = new JObject();
+            obj[NameExitCode] = testResult.ExitCode;
+            obj[NameOutputStandard] = testResult.StandardOutput;
+            obj[NameOutputStandard] = testResult.ErrorOutput;
+            obj[NameResultsFileName] = testResult.ResultsFileName;
+            obj[NameResultsFileContent] = testResult.ResultsFileContent;
+            obj[NameEllapsedSeconds] = testResult.Ellapsed.TotalSeconds;
+            return obj;
+        }
+
+        private static JObject CreateTestSourceData()
+        {
+            var obj = new JObject();
+            obj["MachineName"] = Environment.MachineName;
+            obj["TestRoot"] = "";
+            return obj;
+        }
+    }
+}

src/Tools/Source/RunTests/Program.cs

@@ -33,7 +33,18 @@ internal static int Main(string[] args)
             ITestExecutor testExecutor = new ProcessTestExecutor(options);
             if (options.UseCachedResults)
             {
-                testExecutor = new CachingTestExecutor(options, testExecutor, new LocalDataStorage());
+                // The web caching layer is still being worked on.  For now want to limit it to Roslyn developers
+                // and Jenkins runs by default until we work on this a bit more.  Anyone reading this who wants
+                // to try it out should feel free to opt into this. 
+                IDataStorage dataStorage = new LocalDataStorage();
+                if (StringComparer.OrdinalIgnoreCase.Equals("REDMOND", Environment.UserDomainName) || 
+                    !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("JENKINS_URL")))
+                {
+                    Console.WriteLine("Using web cache");
+                    dataStorage = new WebDataStorage();
+                }
+
+                testExecutor = new CachingTestExecutor(options, testExecutor, dataStorage);
             }
 
             var testRunner = new TestRunner(options, testExecutor);

src/Tools/Source/RunTests/RunTests.csproj

@@ -13,9 +13,13 @@
     <AssemblyName>RunTests</AssemblyName>
     <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
     <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
+    <SignAssembly>false</SignAssembly>
+    <OutDir>$(OutDir)RunTests\</OutDir>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' " />
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' " />
+  <PropertyGroup>
+  </PropertyGroup>
   <ItemGroup>
     <Reference Include="System" />
     <Reference Include="System.Core" />
@@ -26,6 +30,7 @@
     <Compile Include="Cache\ContentUtil.cs" />
     <Compile Include="Cache\CachingTestExecutor.cs" />
     <Compile Include="Cache\ConentFile.cs" />
+    <Compile Include="Cache\WebDataStorage.cs" />
     <Compile Include="Constants.cs" />
     <Compile Include="FileUtil.cs" />
     <Compile Include="Cache\IDataStorage.cs" />
@@ -39,7 +44,10 @@
     <Compile Include="TestRunner.cs" />
     <Compile Include="ConsoleUtil.cs" />
   </ItemGroup>
+  <ItemGroup>
+    <None Include="project.json" />
+  </ItemGroup>
   <ImportGroup Label="Targets">
     <Import Project="..\..\..\..\build\Targets\VSL.Imports.targets" />
   </ImportGroup>
-</Project>
\ No newline at end of file
+</Project>

src/Tools/Source/RunTests/project.json

+{
+  "dependencies": { 
+    "Newtonsoft.Json": "8.0.2",
+    "RestSharp": "105.2.3",
+  },
+  "frameworks": {
+    "net45": { }
+  },
+  "runtimes": {
+    "win7": { },
+    "win7-anycpu": { }
+  }
+}

src/Tools/Source/RunTests/project.lock.json

+{
+  "locked": false,
+  "version": 1,
+  "targets": {
+    ".NETFramework,Version=v4.5": {
+      "Newtonsoft.Json/8.0.2": {
+        "compile": {
+          "lib/net45/Newtonsoft.Json.dll": {}
+        },
+        "runtime": {
+          "lib/net45/Newtonsoft.Json.dll": {}
+        }
+      },
+      "RestSharp/105.2.3": {
+        "compile": {
+          "lib/net45/RestSharp.dll": {}
+        },
+        "runtime": {
+          "lib/net45/RestSharp.dll": {}
+        }
+      }
+    },
+    ".NETFramework,Version=v4.5/win7": {
+      "Newtonsoft.Json/8.0.2": {
+        "compile": {
+          "lib/net45/Newtonsoft.Json.dll": {}
+        },
+        "runtime": {
+          "lib/net45/Newtonsoft.Json.dll": {}
+        }
+      },
+      "RestSharp/105.2.3": {
+        "compile": {
+          "lib/net45/RestSharp.dll": {}
+        },
+        "runtime": {
+          "lib/net45/RestSharp.dll": {}
+        }
+      }
+    },
+    ".NETFramework,Version=v4.5/win7-anycpu": {
+      "Newtonsoft.Json/8.0.2": {
+        "compile": {
+          "lib/net45/Newtonsoft.Json.dll": {}
+        },
+        "runtime": {
+          "lib/net45/Newtonsoft.Json.dll": {}
+        }
+      },
+      "RestSharp/105.2.3": {
+        "compile": {
+          "lib/net45/RestSharp.dll": {}
+        },
+        "runtime": {
+          "lib/net45/RestSharp.dll": {}
+        }
+      }
+    }
+  },
+  "libraries": {
+    "Newtonsoft.Json/8.0.2": {
+      "sha512": "e5yWmEfu68rmtG431zl9N/7PlNKQDIuiDW5MHlEFAZcecakcxrIGnKqrPAtWNILzK2oNanRB5cD150MYhECK3g==",
+      "type": "Package",
+      "files": [
+        "[Content_Types].xml",
+        "_rels/.rels",
+        "lib/net20/Newtonsoft.Json.dll",
+        "lib/net20/Newtonsoft.Json.xml",
+        "lib/net35/Newtonsoft.Json.dll",
+        "lib/net35/Newtonsoft.Json.xml",
+        "lib/net40/Newtonsoft.Json.dll",
+        "lib/net40/Newtonsoft.Json.xml",
+        "lib/net45/Newtonsoft.Json.dll",
+        "lib/net45/Newtonsoft.Json.xml",
+        "lib/portable-net40+sl5+wp80+win8+wpa81/Newtonsoft.Json.dll",
+        "lib/portable-net40+sl5+wp80+win8+wpa81/Newtonsoft.Json.xml",
+        "lib/portable-net45+wp80+win8+wpa81+dnxcore50/Newtonsoft.Json.dll",
+        "lib/portable-net45+wp80+win8+wpa81+dnxcore50/Newtonsoft.Json.xml",
+        "Newtonsoft.Json.nuspec",
+        "package/services/metadata/core-properties/cc8c591d554640789c6a6ef6456f7772.psmdcp",
+        "tools/install.ps1"
+      ]
+    },
+    "RestSharp/105.2.3": {
+      "sha512": "50LkciOclb7Lza9QXqR9reKeUoPlDQiFD4iBdOmd6twOBmTtqEFIOXQgdDdsTHQg4HcGh6EvrM6gN1X72vkJLA==",
+      "type": "Package",
+      "files": [
+        "[Content_Types].xml",
+        "_rels/.rels",
+        "lib/MonoAndroid10/RestSharp.dll",
+        "lib/MonoAndroid10/RestSharp.xml",
+        "lib/MonoTouch10/RestSharp.dll",
+        "lib/MonoTouch10/RestSharp.xml",
+        "lib/net35/RestSharp.dll",
+        "lib/net35/RestSharp.xml",
+        "lib/net4/RestSharp.dll",
+        "lib/net4/RestSharp.xml",
+        "lib/net45/RestSharp.dll",
+        "lib/net45/RestSharp.xml",
+        "lib/net451/RestSharp.dll",
+        "lib/net451/RestSharp.xml",
+        "lib/net452/RestSharp.dll",
+        "lib/net452/RestSharp.xml",
+        "lib/net46/RestSharp.dll",
+        "lib/net46/RestSharp.xml",
+        "lib/net4-client/RestSharp.dll",
+        "lib/net4-client/RestSharp.xml",
+        "lib/sl5/RestSharp.dll",
+        "lib/sl5/RestSharp.xml",
+        "lib/windowsphone8/RestSharp.dll",
+        "lib/windowsphone8/RestSharp.xml",
+        "lib/windowsphone81/RestSharp.dll",
+        "lib/windowsphone81/RestSharp.xml",
+        "lib/Xamarin.iOS10/RestSharp.dll",
+        "lib/Xamarin.iOS10/RestSharp.xml",
+        "package/services/metadata/core-properties/fe62c7f38ace4dbeaf729ff7b329d22a.psmdcp",
+        "readme.txt",
+        "RestSharp.nuspec"
+      ]
+    }
+  },
+  "projectFileDependencyGroups": {
+    "": [
+      "Newtonsoft.Json >= 8.0.2",
+      "RestSharp >= 105.2.3"
+    ],
+    ".NETFramework,Version=v4.5": []
+  }
+}
\ No newline at end of file