diff --git a/src/Test/Perf/Sample.csx b/src/Test/Perf/Sample.csx new file mode 100644 index 0000000000000000000000000000000000000000..bf6491243fd9f78b3c7d65edcac930fc1536cf84 --- /dev/null +++ b/src/Test/Perf/Sample.csx @@ -0,0 +1,6 @@ +#load "util\test_util.csx" +#load "util\Download_util.csx" + +InitUtilities(); + +DownloadTools(); \ No newline at end of file diff --git a/src/Test/Perf/infra/automation.csx b/src/Test/Perf/infra/automation.csx index 38526f2e0b646ea7c346d559bbb147567998466c..b70fee2078f3a561c478189350c8cafa3fae0df5 100644 --- a/src/Test/Perf/infra/automation.csx +++ b/src/Test/Perf/infra/automation.csx @@ -2,6 +2,7 @@ #load "..\util\test_util.csx" #load "..\util\runner_util.csx" +#load "..\util\Download_util.csx" InitUtilities(); @@ -13,5 +14,8 @@ ShellOutVital(Path.Combine(RoslynDirectory(), "Restore.cmd"), "", workingDirecto // Build Roslyn in Release Mode ShellOutVital("msbuild", "./Roslyn.sln /p:Configuration=Release", workingDirectory: RoslynDirectory()); +// Run DownloadTools before using the TraceManager because TraceManager uses the downloaded CPC binaries +DownloadTools(); + // Run run_and_report.csx await RunFile(Path.Combine(MyWorkingDirectory(), "run_and_report.csx")); diff --git a/src/Test/Perf/infra/fetch_build.csx b/src/Test/Perf/infra/fetch_build.csx index 5ea600ea1611947fc106d457901245a9e7d1742d..6dfe59ef66e5735a4ede0454f6d6c039706ab240 100644 --- a/src/Test/Perf/infra/fetch_build.csx +++ b/src/Test/Perf/infra/fetch_build.csx @@ -1,4 +1,5 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +#load "..\util\tools_util.csx" using System; using System.IO; @@ -47,12 +48,3 @@ bool SanityTestPassesForBuild(string buildPath) return logfileNodes.Count == 0; } - -void CopyDirectory(string source, string destination) -{ - var result = ShellOut("Robocopy", $"/s {source} {destination}"); - if (!result.Succeeded) - { - throw new IOException($"Failed to copy \"{source}\" to \"{destination}\"."); - } -} diff --git a/src/Test/Perf/infra/run_and_report.csx b/src/Test/Perf/infra/run_and_report.csx index 8ab5736bf9001b7e2143830888c0a09df50ecb58..8500359db128d18597e15610a040ed4c5d59ff4a 100644 --- a/src/Test/Perf/infra/run_and_report.csx +++ b/src/Test/Perf/infra/run_and_report.csx @@ -23,6 +23,10 @@ if (result) // Move the json file to a file-share File.Copy(elapsedTimeViBenchJsonFilePath, $@"\\vcbench-srv4\benchview\uploads\vibench\{jsonFileName}"); } +else +{ + Log("Conversion from Consumption to csv failed."); +} // Move the traces to mlangfs1 share UploadTraces(GetCPCDirectoryPath(), @"\\mlangfs1\public\basoundr\PerfTraces"); \ No newline at end of file diff --git a/src/Test/Perf/runner.csx b/src/Test/Perf/runner.csx index 51499530072e9bb1478b4cfccbfb288b9950fab2..842eb5d4aeef054f2dd1a78b8f4914f2b85a9734 100644 --- a/src/Test/Perf/runner.csx +++ b/src/Test/Perf/runner.csx @@ -3,7 +3,6 @@ #load ".\util\runner_util.csx" #load ".\util\test_util.csx" #load ".\util\TraceManager_util.csx" -#load ".\util\DownloadCPC_util.csx" using System.Collections.Generic; using System.IO; @@ -11,13 +10,11 @@ using System; InitUtilities(); -var testDirectory = PerfTestDirectory(); +var testDirectory = Path.Combine(MyWorkingDirectory(), "Tests"); var allResults = new List>>>(); var failed = false; -// Run DownloadCPC before using the TraceManager because TraceManager uses the CPC downloaded binaries -DownloadCPC(); var traceManager = new TraceManager(); // Print message at startup @@ -25,11 +22,9 @@ Log("Starting Performance Test Run"); Log("hash: " + StdoutFrom("git", "show --format=\"%h\" HEAD --").Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.None)[0]); Log("time: " + DateTime.Now.ToString()); -// This is an assumption. We can modify it later -var noOfIterations = 3; traceManager.Setup(); -for(int i = 0; i < noOfIterations; ++ i) +for(int i = 0; i < traceManager.Iterations; ++ i) { traceManager.Start(); diff --git a/src/Test/Perf/util/DownloadCPC_util.csx b/src/Test/Perf/util/Download_util.csx similarity index 55% rename from src/Test/Perf/util/DownloadCPC_util.csx rename to src/Test/Perf/util/Download_util.csx index d999e9551924c90ac2ee159717c22adb53e3a59f..addccaaa492c8584dca3619f622b2ed0d5a1dd99 100644 --- a/src/Test/Perf/util/DownloadCPC_util.csx +++ b/src/Test/Perf/util/Download_util.csx @@ -1,9 +1,16 @@ #load "test_util.csx" +#load "tools_util.csx" using System; using System.IO; -public void DownloadCPC() +void DownloadTools() +{ + DownloadCPC(); + DownloadViBenchToJson(); +} + +void DownloadCPC() { var cpcDestinationPath = GetCPCDirectoryPath(); var cpcSourceBinaryLocation = GetCPCSourceBinaryLocation(); @@ -17,4 +24,12 @@ public void DownloadCPC() // Copy CPC from the share to cpcDestinationPath CopyDirectory(cpcSourceBinaryLocation, cpcDestinationPath); +} + +void DownloadViBenchToJson() +{ + var destinationFilePath = GetViBenchToJsonExeFilePath(); + var sourceFile = @"\\mlangfs1\public\basoundr\vibenchcsv2json\ViBenchToJson.exe"; + + File.Copy(sourceFile, destinationFilePath, true); } \ No newline at end of file diff --git a/src/Test/Perf/util/TraceManager_util.csx b/src/Test/Perf/util/TraceManager_util.csx index b039621993e663ea1b8e1048f0e22a499c807cb9..02ef1a57adb638472869b142047e8cf0e38451e3 100644 --- a/src/Test/Perf/util/TraceManager_util.csx +++ b/src/Test/Perf/util/TraceManager_util.csx @@ -11,15 +11,26 @@ public class TraceManager private string _cpcFullPath = "CPC.exe"; private int _startEventAbsoluteInstance = 1; private int _stopEventAbsoluteInstance = 1; + private readonly int _iterations public TraceManager( + int iterations = 3, string cpcFolderPath = @"%SYSTEMDRIVE%\CPC", string scenarioPath = @"%SYSTEMDRIVE%\CPC") { + _iterations = iterations; _cpcFullPath = Path.Combine(Environment.ExpandEnvironmentVariables(cpcFolderPath), "CPC.exe"); _scenarioGenerator = new ScenarioGenerator(Environment.ExpandEnvironmentVariables(scenarioPath)); } + public int Iterations + { + get + { + return _iterations; + } + } + public void Setup() { var processResult = RunProcess(_cpcFullPath, "/Setup /DisableArchive"); @@ -95,7 +106,7 @@ public class TraceManager _stopEventAbsoluteInstance = 1; } - public static ProcessResult RunProcess(string _cpcFullPath, string args) + private ProcessResult RunProcess(string _cpcFullPath, string args) { var startInfo = new ProcessStartInfo(_cpcFullPath, args); startInfo.UseShellExecute = true; diff --git a/src/Test/Perf/util/test_util.csx b/src/Test/Perf/util/test_util.csx index 83a6cc8edd9b085a8eb6a040bb251b4443211958..bc0d5fb45b348a09fe1af35072f2890e8ce19d98 100644 --- a/src/Test/Perf/util/test_util.csx +++ b/src/Test/Perf/util/test_util.csx @@ -81,11 +81,6 @@ string BinDirectory() return Path.Combine(RoslynDirectory(), "Binaries"); } -string PerfTestDirectory() -{ - return Path.Combine(RoslynDirectory(), "src", "Test", "Perf", "Tests"); -} - string BinDebugDirectory() { return Path.Combine(BinDirectory(), "Debug"); @@ -126,11 +121,16 @@ string GetCPCSourceBinaryLocation() return $@"\\mlangfs1\public\basoundr\CpcBinaries"; } +string GetViBenchToJsonExeFilePath() +{ + return Path.Combine(GetCPCDirectoryPath(), "ViBenchToJson.exe"); +} + // // Process spawning and error handling. // -public class ProcessResult +class ProcessResult { public string ExecutablePath {get; set;} public string Args {get; set;} @@ -216,50 +216,6 @@ ProcessResult ShellOut( }; } -ProcessResult ShellOutUsingShellExecute( - string file, - string args, - string workingDirectory = null, - CancellationToken? cancelationToken = null) -{ - var tcs = new TaskCompletionSource(); - var startInfo = new ProcessStartInfo(file, args); - startInfo.UseShellExecute = true; - var process = new Process - { - StartInfo = startInfo - }; - - if (cancelationToken != null) { - cancelationToken.Value.Register(() => process.Kill()); - } - - if (IsVerbose()) { - Log($"running \"{file}\" with arguments \"{args}\" from directory {workingDirectory}"); - } - - process.Start(); - - process.WaitForExit(); - - return new ProcessResult { - ExecutablePath = file, - Args = args, - Code = 0, - StdOut = "", - StdErr = "", - }; -} - -void CopyDirectory(string source, string destination) -{ - var result = ShellOutUsingShellExecute("Robocopy", $"/mir {source} {destination}"); - if (!result.Succeeded) - { - throw new IOException($"Failed to copy \"{source}\" to \"{destination}\"."); - } -} - string StdoutFrom(string program, string args = "") { var result = ShellOut(program, args); if (result.Failed) { diff --git a/src/Test/Perf/util/tools_util.csx b/src/Test/Perf/util/tools_util.csx index adbd56b8cc3285dfd1b8652b76fd791af548bc30..57585539819a9466c16e57601c6fe6d29a99de3d 100644 --- a/src/Test/Perf/util/tools_util.csx +++ b/src/Test/Perf/util/tools_util.csx @@ -1,5 +1,7 @@ #load "test_util.csx" +using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Xml; @@ -11,6 +13,7 @@ bool ConvertConsumptionToCsv(string source, string destination, string requiredM Log("Entering ConvertConsumptionToCsv"); if (!File.Exists(source)) { + Log($"File {source} does not exist"); return false; } @@ -110,7 +113,7 @@ string GetViBenchJsonFromCsv(string compilerTimeCsvFilePath, string execTimeCsvF arguments = arguments.Replace("\r\n", " ").Replace("\n", ""); - ShellOutVital(@"\\mlangfs1\public\basoundr\vibenchcsv2json\ViBenchToJson.exe", arguments); + ShellOutVital(GetViBenchToJsonExeFilePath(), arguments); return outJson; } @@ -139,4 +142,15 @@ void UploadTraces(string sourceFolderPath, string destinationFolderPath) { Log($"sourceFolderPath: {sourceFolderPath} does not exist"); } +} + +void CopyDirectory(string source, string destination, string argument = @"/mir") +{ + var result = ShellOut("Robocopy", $"{argument} {source} {destination}"); + + // Robocopy has a success exit code from 0 - 7 + if (result.Code > 7) + { + throw new IOException($"Failed to copy \"{source}\" to \"{destination}\"."); + } } \ No newline at end of file