diff --git a/src/Test/Perf/infra/TriggerAutomation.bat b/src/Test/Perf/infra/TriggerAutomation.bat new file mode 100644 index 0000000000000000000000000000000000000000..f70e06d5192196bc0e4a071ba2024bfc19635b6c --- /dev/null +++ b/src/Test/Perf/infra/TriggerAutomation.bat @@ -0,0 +1,14 @@ +@echo off + +:: Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +call "%VS140COMNTOOLS%VsDevCmd.bat" + +pushd %~dp0 + +:: Fetch the signed master release binaries and copy it to open binaries release folder +csi fetch_build.csx Master %~dp0..\..\..\..\Binaries\Release + +:: Start Test Automation from the binaries directory +csi %~dp0..\..\..\..\Binaries\Release\Perf\infra\automation.csx + +popd \ No newline at end of file diff --git a/src/Test/Perf/infra/automation.csx b/src/Test/Perf/infra/automation.csx index c149c8af669f93c7bb3285fe0ea08daaa6881ee8..7e6b8880a8a163cc150e87cd407069d6ca7310ec 100644 --- a/src/Test/Perf/infra/automation.csx +++ b/src/Test/Perf/infra/automation.csx @@ -1,25 +1,14 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. #r "./../../Roslyn.Test.Performance.Utilities.dll" -// IsVerbose() -#load "../util/test_util.csx" // RunFile() #load "../util/runner_util.csx" using Roslyn.Test.Performance.Utilities; var directoryUtil = new RelativeDirectory(); -var logger = new ConsoleAndFileLogger(); TestUtilities.InitUtilities(); -// Update the repository -string branch = TestUtilities.StdoutFrom("git", IsVerbose(), logger, "rev-parse --abbrev-ref HEAD"); -TestUtilities.ShellOutVital("git", $"pull origin {branch}", IsVerbose(), logger); -TestUtilities.ShellOutVital(Path.Combine(directoryUtil.RoslynDirectory, "Restore.cmd"), "", IsVerbose(), logger, workingDirectory: directoryUtil.RoslynDirectory); - -// Build Roslyn in Release Mode -TestUtilities.ShellOutVital("msbuild", "./Roslyn.sln /p:Configuration=Release", IsVerbose(), logger, workingDirectory: directoryUtil.RoslynDirectory); - // Install the Vsixes to RoslynPerf hive await RunFile(Path.Combine(directoryUtil.MyWorkingDirectory, "install_vsixes.csx")); diff --git a/src/Test/Perf/infra/build.csx b/src/Test/Perf/infra/build.csx index 2feb1725b15ef582a3af01321907b2d8f9b9071b..e0ecf751ee671a09f3ddbda01e9f4a88ee9868bc 100644 --- a/src/Test/Perf/infra/build.csx +++ b/src/Test/Perf/infra/build.csx @@ -1,6 +1,6 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. // IsVerbose() -#load "../util/test_util.csx" +#load "../util/tools_util.csx" using System.IO; diff --git a/src/Test/Perf/infra/fetch_build.csx b/src/Test/Perf/infra/fetch_build.csx index 05282d99a80d47724c69bb1981a33f2c3e00c2c6..178f1c757ad106b30f3e9014550fa47becbe629a 100644 --- a/src/Test/Perf/infra/fetch_build.csx +++ b/src/Test/Perf/infra/fetch_build.csx @@ -1,7 +1,6 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -// Log() -#load "../util/test_util.csx" +// CopyDirectory() #load "../util/tools_util.csx" using System; @@ -16,7 +15,7 @@ var destinationFolder = Args.Count() == 2 ? Args[1] : @"C:\Roslyn\Binaries\Relea var sourceFolder = $@"\\cpvsbuild\drops\Roslyn\Roslyn-{branch}-Signed-Release"; string latestBuild = null; -foreach (var folder in Directory.GetFiles(sourceFolder, "????????.?").Reverse()) +foreach (var folder in Directory.GetDirectories(sourceFolder, "????????.?").Reverse()) { if (SanityTestPassesForBuild(folder)) { @@ -31,7 +30,6 @@ if (latestBuild == null) } var latestBuildFolder = Path.Combine(sourceFolder, latestBuild); -Log($"Fetching build \"{latestBuildFolder}\"."); Directory.Delete(destinationFolder, recursive: true); diff --git a/src/Test/Perf/infra/install.csx b/src/Test/Perf/infra/install.csx index c20f54d54b560e654afdc4fe21b88a1b5f41ade8..be494930776040875135756602ef8ede868bb6e7 100644 --- a/src/Test/Perf/infra/install.csx +++ b/src/Test/Perf/infra/install.csx @@ -3,7 +3,7 @@ #r "./../../Roslyn.Test.Performance.Utilities.dll" // IsVerbose() -#load "../util/test_util.csx" +#load "../util/tools_util.csx" #load "./assemblies.csx" #load "./ngen.csx" diff --git a/src/Test/Perf/infra/install_vsixes.csx b/src/Test/Perf/infra/install_vsixes.csx index 6c53902d5e81359e2d22e05e2d065cc31ed25703..8e49ba7c1f1521b5b648b404f974e21900428caa 100644 --- a/src/Test/Perf/infra/install_vsixes.csx +++ b/src/Test/Perf/infra/install_vsixes.csx @@ -2,7 +2,7 @@ #r "./../../Roslyn.Test.Performance.Utilities.dll" // IsVerbose() -#load "../util/test_util.csx" +#load "../util/tools_util.csx" using System.IO; using Roslyn.Test.Performance.Utilities; diff --git a/src/Test/Perf/infra/ngen.csx b/src/Test/Perf/infra/ngen.csx index 8663adf688df66f9c28322dbd3153915862fb5e4..26251bb4ada35242b1d3b1ab634a2478e03abe8c 100644 --- a/src/Test/Perf/infra/ngen.csx +++ b/src/Test/Perf/infra/ngen.csx @@ -2,7 +2,7 @@ // StdoutFrom() // Log() -#load "../util/test_util.csx" +#load "../util/tools_util.csx" using System.IO; diff --git a/src/Test/Perf/infra/run_and_report.csx b/src/Test/Perf/infra/run_and_report.csx index 67ed0dfa399fff9fdc67e22d712335dcd7e45fc5..601cc0558e6b3ce861006fb10378e29b6f3db1a8 100644 --- a/src/Test/Perf/infra/run_and_report.csx +++ b/src/Test/Perf/infra/run_and_report.csx @@ -2,7 +2,7 @@ #r "./../../Roslyn.Test.Performance.Utilities.dll" // IsVerbose() -#load "../util/test_util.csx" +#load "../util/tools_util.csx" // RunFile() #load "../util/runner_util.csx" diff --git a/src/Test/Perf/runner.csx b/src/Test/Perf/runner.csx index 00c33af3703dedb6b1fad50afdbfb28a91263d65..cd41bc83e2a5955f741b44360df6adf71edb73d9 100644 --- a/src/Test/Perf/runner.csx +++ b/src/Test/Perf/runner.csx @@ -6,7 +6,7 @@ #load "./util/runner_util.csx" // Log() // StdoutFrom() -#load "./util/test_util.csx" +#load "./util/tools_util.csx" using System.Collections.Generic; using System.IO; diff --git a/src/Test/Perf/tests/csharp/csharp_compiler.csx b/src/Test/Perf/tests/csharp/csharp_compiler.csx index e25e67f714e6336c367fed85bd8f1b5ddbad0d6b..1a60102a02de223def9fea8a850b6ade87fdd1c8 100644 --- a/src/Test/Perf/tests/csharp/csharp_compiler.csx +++ b/src/Test/Perf/tests/csharp/csharp_compiler.csx @@ -2,8 +2,9 @@ #r "../../../Roslyn.Test.Performance.Utilities.dll" // TestThisPlease() -// IsVerbose() #load "../../util/test_util.csx" +// IsVerbose() +#load "../../util/tools_util.csx" using System.IO; using System.Collections.Generic; diff --git a/src/Test/Perf/tests/helloworld/hello_world.csx b/src/Test/Perf/tests/helloworld/hello_world.csx index 55b78ebc659826f95ac2c347a42bd5baf271acea..92e8c85b05e8a5e146aafeedfd4c6e2b1ed680da 100644 --- a/src/Test/Perf/tests/helloworld/hello_world.csx +++ b/src/Test/Perf/tests/helloworld/hello_world.csx @@ -3,8 +3,9 @@ #r "../../../Roslyn.Test.Performance.Utilities.dll" // TestThisPlease() +#load "../../util/test_util.csx" // IsVerbose() -#load "./../../util/test_util.csx" +#load "../../util/tools_util.csx" using System.IO; using Roslyn.Test.Performance.Utilities; diff --git a/src/Test/Perf/util/RelativeDirectory.cs b/src/Test/Perf/util/RelativeDirectory.cs index d75e21db4c5f9ee5cfb8e43fea8278873705ab6a..7cb5c6dd7c94f46ab07eb74f0abde45d5b00b6aa 100644 --- a/src/Test/Perf/util/RelativeDirectory.cs +++ b/src/Test/Perf/util/RelativeDirectory.cs @@ -89,6 +89,8 @@ public string MyBinaries() throw new Exception("Couldn't find binaries. Are you running from the binaries directory?"); } + public string TaoPath => Path.Combine(MyBinaries(), "Tao"); + public string PerfDirectory => Path.Combine(RoslynDirectory, "src", "Test", "Perf"); public string BinDirectory => Path.Combine(RoslynDirectory, "Binaries"); diff --git a/src/Test/Perf/util/TraceManagerFactory.cs b/src/Test/Perf/util/TraceManagerFactory.cs index 82704ed1a12ca488bc876a76f94d5434e324ba2c..e705c3e69ece1b363b0378b06b57a690d4863d74 100644 --- a/src/Test/Perf/util/TraceManagerFactory.cs +++ b/src/Test/Perf/util/TraceManagerFactory.cs @@ -10,7 +10,7 @@ public static ITraceManager GetTraceManager() var scenarioPath = TestUtilities.GetCPCDirectoryPath(); if (File.Exists(cpcFullPath)) { - return new TraceManager(cpcFullPath, scenarioPath, verbose: false, logger: null); + return new TraceManager(cpcFullPath, scenarioPath, verbose: true, logger: null); } else { diff --git a/src/Test/Perf/util/test_util.csx b/src/Test/Perf/util/test_util.csx index 905232c4495daab181e972deb2cf1c07b4761d88..90125efc10c6e5753419be82699ce29f51921408 100644 --- a/src/Test/Perf/util/test_util.csx +++ b/src/Test/Perf/util/test_util.csx @@ -37,149 +37,3 @@ static void TestThisPlease(params PerfTest[] tests) } } -// -// Arguments -// - -// This is due to a design decision in csi that has Args non-static. -// Non-static variables are impossible to read inide of nested classes. -static IEnumerable StaticArgs = null; -StaticArgs = Args; - -/// Returns the path to log file if one exists. -/// Returns null otherwise. -static string LogFile() -{ - var key = "--log="; - return (from arg in StaticArgs where arg.StartsWith(key) select arg.Substring(key.Length)).FirstOrDefault(); -} - -/// Returns true if --verbosity is passed on the command line -static bool IsVerbose() -{ - return StaticArgs.Contains("--verbose"); -} - -static bool IsRunFromRunner() -{ - return StaticArgs.Contains("--from-runner"); -} - -// -// Process spawning and error handling. -// - -class ProcessResult -{ - public string ExecutablePath {get; set;} - public string Args {get; set;} - public int Code {get; set;} - public string StdOut {get; set;} - public string StdErr {get; set;} - - public bool Failed => Code != 0; - public bool Succeeded => !Failed; -} - -static ProcessResult ShellOut( - string file, - string args, - string workingDirectory, - CancellationToken? cancelationToken = null) -{ - var tcs = new TaskCompletionSource(); - var startInfo = new ProcessStartInfo(file, args); - startInfo.RedirectStandardOutput = true; - startInfo.RedirectStandardError = true; - startInfo.UseShellExecute = false; - startInfo.WorkingDirectory = workingDirectory; - - - var process = new Process - { - StartInfo = startInfo, - EnableRaisingEvents = true, - }; - - if (cancelationToken != null) - { - cancelationToken.Value.Register(() => process.Kill()); - } - - if (IsVerbose()) - { - Log($"running \"{file}\" with arguments \"{args}\" from directory {workingDirectory}"); - } - - process.Start(); - - var output = new StringWriter(); - var error = new StringWriter(); - - process.OutputDataReceived += (s, e) => { - if (!String.IsNullOrEmpty(e.Data)) - { - output.WriteLine(e.Data); - } - }; - - process.ErrorDataReceived += (s, e) => { - if (!String.IsNullOrEmpty(e.Data)) - { - error.WriteLine(e.Data); - } - }; - - process.BeginOutputReadLine(); - process.BeginErrorReadLine(); - process.WaitForExit(); - - return new ProcessResult { - ExecutablePath = file, - Args = args, - Code = process.ExitCode, - StdOut = output.ToString(), - StdErr = error.ToString(), - }; -} - -string StdoutFrom(string program, string args = "", string workingDirectory = null) -{ - if (workingDirectory == null) { - var directoryInfo = new RelativeDirectory(); - workingDirectory = directoryInfo.MyTempDirectory; - } - - var result = ShellOut(program, args, workingDirectory); - if (result.Failed) - { - LogProcessResult(result); - throw new Exception("Shelling out failed"); - } - return result.StdOut.Trim(); -} - -/// Logs a message. -/// -/// The actual implementation of this method may change depending on -/// if the script is being run standalone or through the test runner. -static void Log(string info) -{ - System.Console.WriteLine(info); - var log = LogFile(); - if (log != null) - { - File.AppendAllText(log, info + System.Environment.NewLine); - } -} - -/// Logs the result of a finished process -static void LogProcessResult(ProcessResult result) -{ - Log(String.Format("The process \"{0}\" {1} with code {2}", - $"{result.ExecutablePath} {result.Args}", - result.Failed ? "failed" : "succeeded", - result.Code)); - Log($"Standard Out:\n{result.StdOut}"); - Log($"\nStandard Error:\n{result.StdErr}"); -} diff --git a/src/Test/Perf/util/tools_util.csx b/src/Test/Perf/util/tools_util.csx index 55070ba50a9e9d840d870a6d92eaab2d3c89a0fa..6f47a4a546020aeafda35939c659082ec79a682c 100644 --- a/src/Test/Perf/util/tools_util.csx +++ b/src/Test/Perf/util/tools_util.csx @@ -1,10 +1,17 @@ -// ShellOut() -#load "test_util.csx" +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +#r "System.IO.Compression.FileSystem" using System; using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; using System.IO; +using System.IO.Compression; +using System.Net; using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading.Tasks; +using System.Threading; using System.Xml; void CopyDirectory(string source, string destination, string argument = @"/mir") @@ -17,3 +24,146 @@ void CopyDirectory(string source, string destination, string argument = @"/mir") throw new IOException($"Failed to copy \"{source}\" to \"{destination}\"."); } } + +// +// Arguments +// + +// This is due to a design decision in csi that has Args non-static. +// Non-static variables are impossible to read inide of nested classes. +static IEnumerable StaticArgs = null; +StaticArgs = Args; + +/// Returns the path to log file if one exists. +/// Returns null otherwise. +static string LogFile() +{ + var key = "--log="; + return (from arg in StaticArgs where arg.StartsWith(key) select arg.Substring(key.Length)).FirstOrDefault(); +} + +/// Returns true if --verbosity is passed on the command line +static bool IsVerbose() +{ + return StaticArgs.Contains("--verbose"); +} + +static bool IsRunFromRunner() +{ + return StaticArgs.Contains("--from-runner"); +} + +// +// Process spawning and error handling. +// + +class ProcessResult +{ + public string ExecutablePath { get; set; } + public string Args { get; set; } + public int Code { get; set; } + public string StdOut { get; set; } + public string StdErr { get; set; } + + public bool Failed => Code != 0; + public bool Succeeded => !Failed; +} + +static ProcessResult ShellOut( + string file, + string args, + string workingDirectory, + CancellationToken? cancelationToken = null) +{ + var tcs = new TaskCompletionSource(); + var startInfo = new ProcessStartInfo(file, args); + startInfo.RedirectStandardOutput = true; + startInfo.RedirectStandardError = true; + startInfo.UseShellExecute = false; + startInfo.WorkingDirectory = workingDirectory; + + + var process = new Process + { + StartInfo = startInfo, + EnableRaisingEvents = true, + }; + + if (cancelationToken != null) + { + cancelationToken.Value.Register(() => process.Kill()); + } + + if (IsVerbose()) + { + Log($"running \"{file}\" with arguments \"{args}\" from directory {workingDirectory}"); + } + + process.Start(); + + var output = new StringWriter(); + var error = new StringWriter(); + + process.OutputDataReceived += (s, e) => { + if (!String.IsNullOrEmpty(e.Data)) + { + output.WriteLine(e.Data); + } + }; + + process.ErrorDataReceived += (s, e) => { + if (!String.IsNullOrEmpty(e.Data)) + { + error.WriteLine(e.Data); + } + }; + + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + process.WaitForExit(); + + return new ProcessResult + { + ExecutablePath = file, + Args = args, + Code = process.ExitCode, + StdOut = output.ToString(), + StdErr = error.ToString(), + }; +} + +string StdoutFrom(string program, string args = "", string workingDirectory = "") +{ + var result = ShellOut(program, args, workingDirectory); + if (result.Failed) + { + LogProcessResult(result); + throw new Exception("Shelling out failed"); + } + return result.StdOut.Trim(); +} + +/// Logs a message. +/// +/// The actual implementation of this method may change depending on +/// if the script is being run standalone or through the test runner. +static void Log(string info) +{ + System.Console.WriteLine(info); + var log = LogFile(); + if (log != null) + { + File.AppendAllText(log, info + System.Environment.NewLine); + } +} + +/// Logs the result of a finished process +static void LogProcessResult(ProcessResult result) +{ + Log(String.Format("The process \"{0}\" {1} with code {2}", + $"{result.ExecutablePath} {result.Args}", + result.Failed ? "failed" : "succeeded", + result.Code)); + Log($"Standard Out:\n{result.StdOut}"); + Log($"\nStandard Error:\n{result.StdErr}"); +} diff --git a/src/Test/PerformanceTesting.csproj b/src/Test/PerformanceTesting.csproj index 29e6f7920c8db9d2e9e2df156e61e90600ac6a9b..93c6c63a3fdc46d21eaeacc8168d456082031932 100644 --- a/src/Test/PerformanceTesting.csproj +++ b/src/Test/PerformanceTesting.csproj @@ -98,6 +98,9 @@ PreserveNewest + + PreserveNewest + PreserveNewest @@ -123,7 +126,7 @@ - +