Merge pull request #11052 from basoundr/FixAutomationToFetch

Fix automation to fetch

src/Test/Perf/infra/TriggerAutomation.bat

+@echo off
+
+:: Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+call "%VS140COMNTOOLS%VsDevCmd.bat"
+
+pushd %~dp0
+
+:: Fetch the signed master release binaries and copy it to open binaries release folder
+csi fetch_build.csx Master %~dp0..\..\..\..\Binaries\Release
+
+:: Start Test Automation from the binaries directory
+csi %~dp0..\..\..\..\Binaries\Release\Perf\infra\automation.csx
+
+popd
\ No newline at end of file

src/Test/Perf/infra/automation.csx

 // Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
 #r "./../../Roslyn.Test.Performance.Utilities.dll"
 
-// IsVerbose()
-#load "../util/test_util.csx"
 // RunFile()
 #load "../util/runner_util.csx"
 
 using Roslyn.Test.Performance.Utilities;
 
 var directoryUtil = new RelativeDirectory();
-var logger = new ConsoleAndFileLogger();
 TestUtilities.InitUtilities();
 
-// Update the repository
-string branch = TestUtilities.StdoutFrom("git", IsVerbose(), logger, "rev-parse --abbrev-ref HEAD");
-TestUtilities.ShellOutVital("git", $"pull origin {branch}", IsVerbose(), logger);
-TestUtilities.ShellOutVital(Path.Combine(directoryUtil.RoslynDirectory, "Restore.cmd"), "", IsVerbose(), logger, workingDirectory: directoryUtil.RoslynDirectory);
-
-// Build Roslyn in Release Mode
-TestUtilities.ShellOutVital("msbuild", "./Roslyn.sln /p:Configuration=Release", IsVerbose(), logger, workingDirectory: directoryUtil.RoslynDirectory);
-
 // Install the Vsixes to RoslynPerf hive
 await RunFile(Path.Combine(directoryUtil.MyWorkingDirectory, "install_vsixes.csx"));
 

src/Test/Perf/infra/build.csx

 // Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
 // IsVerbose()
-#load "../util/test_util.csx"
+#load "../util/tools_util.csx"
 
 using System.IO;
 

src/Test/Perf/infra/fetch_build.csx

 // Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
 
-// Log()
-#load "../util/test_util.csx"
+// CopyDirectory()
 #load "../util/tools_util.csx"
 
 using System;
@@ -16,7 +15,7 @@ var destinationFolder = Args.Count() == 2 ? Args[1] : @"C:\Roslyn\Binaries\Relea
 var sourceFolder = $@"\\cpvsbuild\drops\Roslyn\Roslyn-{branch}-Signed-Release";
 
 string latestBuild = null;
-foreach (var folder in Directory.GetFiles(sourceFolder, "????????.?").Reverse())
+foreach (var folder in Directory.GetDirectories(sourceFolder, "????????.?").Reverse())
 {
     if (SanityTestPassesForBuild(folder))
     {
@@ -31,7 +30,6 @@ if (latestBuild == null)
 }
 
 var latestBuildFolder = Path.Combine(sourceFolder, latestBuild);
-Log($"Fetching build \"{latestBuildFolder}\".");
 
 Directory.Delete(destinationFolder, recursive: true);
 

src/Test/Perf/infra/install.csx

@@ -3,7 +3,7 @@
 #r "./../../Roslyn.Test.Performance.Utilities.dll"
 
 // IsVerbose()
-#load "../util/test_util.csx"
+#load "../util/tools_util.csx"
 #load "./assemblies.csx"
 #load "./ngen.csx"
 

src/Test/Perf/infra/install_vsixes.csx

@@ -2,7 +2,7 @@
 #r "./../../Roslyn.Test.Performance.Utilities.dll"
 
 // IsVerbose()
-#load "../util/test_util.csx"
+#load "../util/tools_util.csx"
 
 using System.IO;
 using Roslyn.Test.Performance.Utilities;

src/Test/Perf/infra/ngen.csx

@@ -2,7 +2,7 @@
 
 // StdoutFrom()
 // Log()
-#load "../util/test_util.csx"
+#load "../util/tools_util.csx"
 
 using System.IO;
 

src/Test/Perf/infra/run_and_report.csx

@@ -2,7 +2,7 @@
 #r "./../../Roslyn.Test.Performance.Utilities.dll"
 
 // IsVerbose()
-#load "../util/test_util.csx"
+#load "../util/tools_util.csx"
 // RunFile()
 #load "../util/runner_util.csx"
 

src/Test/Perf/runner.csx

@@ -6,7 +6,7 @@
 #load "./util/runner_util.csx"
 // Log()
 // StdoutFrom()
-#load "./util/test_util.csx"
+#load "./util/tools_util.csx"
 
 using System.Collections.Generic;
 using System.IO;

src/Test/Perf/tests/csharp/csharp_compiler.csx

@@ -2,8 +2,9 @@
 #r "../../../Roslyn.Test.Performance.Utilities.dll"
 
 // TestThisPlease()
-// IsVerbose()
 #load "../../util/test_util.csx"
+// IsVerbose()
+#load "../../util/tools_util.csx"
 
 using System.IO;
 using System.Collections.Generic;

src/Test/Perf/tests/helloworld/hello_world.csx

@@ -3,8 +3,9 @@
 #r "../../../Roslyn.Test.Performance.Utilities.dll"
 
 // TestThisPlease()
+#load "../../util/test_util.csx"
 // IsVerbose()
-#load "./../../util/test_util.csx"
+#load "../../util/tools_util.csx"
 
 using System.IO;
 using Roslyn.Test.Performance.Utilities;

src/Test/Perf/util/RelativeDirectory.cs

@@ -89,6 +89,8 @@ public string MyBinaries()
             throw new Exception("Couldn't find binaries. Are you running from the binaries directory?");
         }
 
+        public string TaoPath => Path.Combine(MyBinaries(), "Tao");
+
         public string PerfDirectory => Path.Combine(RoslynDirectory, "src", "Test", "Perf");
 
         public string BinDirectory => Path.Combine(RoslynDirectory, "Binaries");

src/Test/Perf/util/TraceManagerFactory.cs

@@ -10,7 +10,7 @@ public static ITraceManager GetTraceManager()
             var scenarioPath = TestUtilities.GetCPCDirectoryPath();
             if (File.Exists(cpcFullPath))
             {
-                return new TraceManager(cpcFullPath, scenarioPath, verbose: false, logger: null);
+                return new TraceManager(cpcFullPath, scenarioPath, verbose: true, logger: null);
             }
             else
             {

src/Test/Perf/util/test_util.csx

@@ -37,149 +37,3 @@ static void TestThisPlease(params PerfTest[] tests)
     }
 }
 
-//
-// Arguments
-//
-
-// This is due to a design decision in csi that has Args non-static.
-// Non-static variables are impossible to read inide of nested classes.
-static IEnumerable<string> StaticArgs = null;
-StaticArgs = Args;
-
-/// Returns the path to log file if one exists.
-/// Returns null otherwise.
-static string LogFile()
-{
-    var key = "--log=";
-    return (from arg in StaticArgs where arg.StartsWith(key) select arg.Substring(key.Length)).FirstOrDefault();
-}
-
-/// Returns true if --verbosity is passed on the command line
-static bool IsVerbose()
-{
-    return StaticArgs.Contains("--verbose");
-}
-
-static bool IsRunFromRunner() 
-{
-    return StaticArgs.Contains("--from-runner");
-}
-
-//
-// Process spawning and error handling.
-//
-
-class ProcessResult
-{
-    public string ExecutablePath {get; set;}
-    public string Args {get; set;}
-    public int Code {get; set;}
-    public string StdOut {get; set;}
-    public string StdErr {get; set;}
-
-    public bool Failed => Code != 0;
-    public bool Succeeded => !Failed;
-}
-
-static ProcessResult ShellOut(
-        string file,
-        string args,
-        string workingDirectory,
-        CancellationToken? cancelationToken = null)
-{
-    var tcs = new TaskCompletionSource<ProcessResult>();
-    var startInfo = new ProcessStartInfo(file, args);
-    startInfo.RedirectStandardOutput = true;
-    startInfo.RedirectStandardError = true;
-    startInfo.UseShellExecute = false;
-    startInfo.WorkingDirectory = workingDirectory;
-    
-    
-    var process = new Process
-    {
-        StartInfo = startInfo,
-        EnableRaisingEvents = true,
-    };
-
-    if (cancelationToken != null)
-    {
-        cancelationToken.Value.Register(() => process.Kill());
-    }
-
-    if (IsVerbose())
-    {
-        Log($"running \"{file}\" with arguments \"{args}\" from directory {workingDirectory}");
-    }
-
-    process.Start();
-
-    var output = new StringWriter();
-    var error = new StringWriter();
-
-    process.OutputDataReceived += (s, e) => {
-        if (!String.IsNullOrEmpty(e.Data))
-        {
-            output.WriteLine(e.Data);
-        }
-    };
-
-    process.ErrorDataReceived += (s, e) => {
-        if (!String.IsNullOrEmpty(e.Data))
-        {
-            error.WriteLine(e.Data);
-        }
-    };
-
-    process.BeginOutputReadLine();
-    process.BeginErrorReadLine();
-    process.WaitForExit();
-
-    return new ProcessResult {
-        ExecutablePath = file,
-        Args = args,
-        Code = process.ExitCode,
-        StdOut = output.ToString(),
-        StdErr = error.ToString(),
-    };
-}
-
-string StdoutFrom(string program, string args = "", string workingDirectory = null)
-{
-    if (workingDirectory == null) {
-        var directoryInfo = new RelativeDirectory();
-        workingDirectory = directoryInfo.MyTempDirectory;
-    }
-    
-    var result = ShellOut(program, args, workingDirectory);
-    if (result.Failed)
-    {
-        LogProcessResult(result);
-        throw new Exception("Shelling out failed");
-    }
-    return result.StdOut.Trim();
-}
-
-/// Logs a message.
-///
-/// The actual implementation of this method may change depending on
-/// if the script is being run standalone or through the test runner.
-static void Log(string info)
-{
-    System.Console.WriteLine(info);
-    var log = LogFile();
-    if (log != null)
-    {
-        File.AppendAllText(log, info + System.Environment.NewLine);
-    }
-}
-
-/// Logs the result of a finished process
-static void LogProcessResult(ProcessResult result)
-{
-    Log(String.Format("The process \"{0}\" {1} with code {2}",
-        $"{result.ExecutablePath} {result.Args}",
-        result.Failed ? "failed" : "succeeded",
-        result.Code));
-    Log($"Standard Out:\n{result.StdOut}");
-    Log($"\nStandard Error:\n{result.StdErr}");
-}

src/Test/Perf/util/tools_util.csx

-// ShellOut()
-#load "test_util.csx"
+// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+#r "System.IO.Compression.FileSystem"
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
+using System.Globalization;
 using System.IO;
+using System.IO.Compression;
+using System.Net;
 using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Threading.Tasks;
+using System.Threading;
 using System.Xml;
 
 void CopyDirectory(string source, string destination, string argument = @"/mir")
@@ -17,3 +24,146 @@ void CopyDirectory(string source, string destination, string argument = @"/mir")
         throw new IOException($"Failed to copy \"{source}\" to \"{destination}\".");
     }
 }
+
+//
+// Arguments
+//
+
+// This is due to a design decision in csi that has Args non-static.
+// Non-static variables are impossible to read inide of nested classes.
+static IEnumerable<string> StaticArgs = null;
+StaticArgs = Args;
+
+/// Returns the path to log file if one exists.
+/// Returns null otherwise.
+static string LogFile()
+{
+    var key = "--log=";
+    return (from arg in StaticArgs where arg.StartsWith(key) select arg.Substring(key.Length)).FirstOrDefault();
+}
+
+/// Returns true if --verbosity is passed on the command line
+static bool IsVerbose()
+{
+    return StaticArgs.Contains("--verbose");
+}
+
+static bool IsRunFromRunner()
+{
+    return StaticArgs.Contains("--from-runner");
+}
+
+//
+// Process spawning and error handling.
+//
+
+class ProcessResult
+{
+    public string ExecutablePath { get; set; }
+    public string Args { get; set; }
+    public int Code { get; set; }
+    public string StdOut { get; set; }
+    public string StdErr { get; set; }
+
+    public bool Failed => Code != 0;
+    public bool Succeeded => !Failed;
+}
+
+static ProcessResult ShellOut(
+        string file,
+        string args,
+        string workingDirectory,
+        CancellationToken? cancelationToken = null)
+{
+    var tcs = new TaskCompletionSource<ProcessResult>();
+    var startInfo = new ProcessStartInfo(file, args);
+    startInfo.RedirectStandardOutput = true;
+    startInfo.RedirectStandardError = true;
+    startInfo.UseShellExecute = false;
+    startInfo.WorkingDirectory = workingDirectory;
+
+
+    var process = new Process
+    {
+        StartInfo = startInfo,
+        EnableRaisingEvents = true,
+    };
+
+    if (cancelationToken != null)
+    {
+        cancelationToken.Value.Register(() => process.Kill());
+    }
+
+    if (IsVerbose())
+    {
+        Log($"running \"{file}\" with arguments \"{args}\" from directory {workingDirectory}");
+    }
+
+    process.Start();
+
+    var output = new StringWriter();
+    var error = new StringWriter();
+
+    process.OutputDataReceived += (s, e) => {
+        if (!String.IsNullOrEmpty(e.Data))
+        {
+            output.WriteLine(e.Data);
+        }
+    };
+
+    process.ErrorDataReceived += (s, e) => {
+        if (!String.IsNullOrEmpty(e.Data))
+        {
+            error.WriteLine(e.Data);
+        }
+    };
+
+    process.BeginOutputReadLine();
+    process.BeginErrorReadLine();
+    process.WaitForExit();
+
+    return new ProcessResult
+    {
+        ExecutablePath = file,
+        Args = args,
+        Code = process.ExitCode,
+        StdOut = output.ToString(),
+        StdErr = error.ToString(),
+    };
+}
+
+string StdoutFrom(string program, string args = "", string workingDirectory = "")
+{
+    var result = ShellOut(program, args, workingDirectory);
+    if (result.Failed)
+    {
+        LogProcessResult(result);
+        throw new Exception("Shelling out failed");
+    }
+    return result.StdOut.Trim();
+}
+
+/// Logs a message.
+///
+/// The actual implementation of this method may change depending on
+/// if the script is being run standalone or through the test runner.
+static void Log(string info)
+{
+    System.Console.WriteLine(info);
+    var log = LogFile();
+    if (log != null)
+    {
+        File.AppendAllText(log, info + System.Environment.NewLine);
+    }
+}
+
+/// Logs the result of a finished process
+static void LogProcessResult(ProcessResult result)
+{
+    Log(String.Format("The process \"{0}\" {1} with code {2}",
+        $"{result.ExecutablePath} {result.Args}",
+        result.Failed ? "failed" : "succeeded",
+        result.Code));
+    Log($"Standard Out:\n{result.StdOut}");
+    Log($"\nStandard Error:\n{result.StdErr}");
+}

src/Test/PerformanceTesting.csproj

@@ -98,6 +98,9 @@
     <Content Include="Perf\tests\helloworld\hello_world.csx">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+    <None Include="Perf\infra\TriggerAutomation.bat">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
     <None Include="Perf\tests\csharp\csharp_compiler.csx">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
@@ -123,7 +126,7 @@
     <Compile Include="Perf\util\ILogger.cs" />
     <Compile Include="Perf\util\ITraceManager.cs" />
     <Compile Include="Perf\util\NoOpTraceManager.cs" />
-    <Compile Include="Perf\util\ScenarioGenerator.cs"/>
+    <Compile Include="Perf\util\ScenarioGenerator.cs" />
     <Compile Include="Perf\util\TestUtilities.cs" />
     <Compile Include="Perf\util\TraceManager.cs" />
     <Compile Include="Perf\util\TraceManagerFactory.cs" />