Fix CPC

Earlier the perf test runner using CPC was broken. This change fixes that.

With this change, we will be running CPC a little different than what we used to.
Earlier, we run CPC for 'n' of iterations and the first iteration should to be a
warmup iteration. Each iteration runs all the tests once.

Now, we run CPC for 'n' iterations for each test and 'n' is defined by each test.
The first iteration is, again, a warmup iteration and the rest are considered for the
metrics calculation. With this model, each iteration will run just one test instead of
running all the tests in an iteration. This helps in making the warmup iteration, a
true warmup iteration.

Minor changes:
Since the test csx returns an object now, the test related information has been
moved to the corresponding test csx files.

src/Test/Perf/infra/automation.csx

@@ -14,8 +14,5 @@ ShellOutVital(Path.Combine(directoryUtil.RoslynDirectory, "Restore.cmd"), "", wo
 // Build Roslyn in Release Mode
 ShellOutVital("msbuild", "./Roslyn.sln /p:Configuration=Release", workingDirectory: directoryUtil.RoslynDirectory);
 
-// Run DownloadTools before using the TraceManager because TraceManager uses the downloaded CPC binaries
-DownloadTools();
-
 // Run run_and_report.csx
 await RunFile(Path.Combine(directoryUtil.MyWorkingDirectory, "run_and_report.csx"));

src/Test/Perf/runner.csx

@@ -9,7 +9,7 @@ using System.IO;
 using System;
 
 var directoryInfo = new RelativeDirectory();
-var testDirectory = Path.Combine(directoryInfo.MyWorkingDirectory, "Tests");
+var testDirectory = Path.Combine(directoryInfo.MyWorkingDirectory, "tests");
 
 // Print message at startup
 Log("Starting Performance Test Run");
@@ -29,45 +29,31 @@ foreach (var script in GetAllCsxRecursive(testDirectory))
 }
 
 var traceManager = TraceManagerFactory.GetTraceManager();
-traceManager.Setup();
-for (int i = 0; i < traceManager.Iterations; i++)
-{
-    traceManager.Start();
-    foreach (dynamic test in testInstances)
-    {
-        test.Setup();
-        traceManager.StartScenario(test.Name, test.MeasuredProc);
-        traceManager.StartEvent();
-        test.Test();
-        traceManager.EndEvent();
-        traceManager.EndScenario();
-    }
-    traceManager.EndScenarios();
-    traceManager.WriteScenariosFileToDisk();
-    traceManager.Stop();
-    traceManager.ResetScenarioGenerator();
-}
-traceManager.Cleanup();
-
-/*
-var traceManager = TraceManagerFactory.GetTraceManager();
-traceManager.Setup();
-// Run each of the tests
-foreach (dynamic test in testInstances) 
+traceManager.Initialize();
+foreach (dynamic test in testInstances)
 {
     test.Setup();
-    traceManager.Start();
-    for (int i = 0; i < test.Iterations; i++) 
+    traceManager.Setup();
+    
+    var iterations = traceManager.HasWarmUpIteration ? 
+                     test.Iterations + 1 :
+                     test.Iterations;
+                     
+    for (int i = 0; i < iterations; i++)
     {
-        traceManager.StartScenario("temp" + i, "csc");
+        traceManager.StartScenarios();
+        traceManager.Start();
+        traceManager.StartScenario(test.Name + i, test.MeasuredProc);
         traceManager.StartEvent();
         test.Test();
         traceManager.EndEvent();
         traceManager.EndScenario();
+        
+        traceManager.EndScenarios();
+        traceManager.WriteScenariosFileToDisk();
+        traceManager.Stop();
+        traceManager.ResetScenarioGenerator();
     }
-    traceManager.EndScenarios();
-    traceManager.WriteScenariosFileToDisk();
-    traceManager.Stop();
+
     traceManager.Cleanup();
 }
-*/

src/Test/Perf/tests/csharp/csharp_compiler.csx

@@ -27,9 +27,9 @@ class CSharpCompilerTest: PerfTest
         ShellOutVital(ReleaseCscPath, args, executeInDirectory);
     }
     
-    public override int Iterations => 1;
+    public override int Iterations => 2;
     public override string Name => "csharp " + _rspFile;
-    public override string MeasuredProc => "csc.exe";
+    public override string MeasuredProc => "csc";
 }
 
 TestThisPlease(

src/Test/Perf/tests/helloworld/hello_world.csx

@@ -21,9 +21,9 @@ class HelloWorldTest: PerfTest
         ShellOutVital(ReleaseCscPath, _pathToHelloWorld + " /out:" + _pathToOutput);
     }
     
-    public override int Iterations => 1;
+    public override int Iterations => 2;
     public override string Name => "hello world";
-    public override string MeasuredProc => "csc.exe";
+    public override string MeasuredProc => "csc";
 }
 
 TestThisPlease(new HelloWorldTest());

src/Test/Perf/util/scenario_generator_util.csx

@@ -26,7 +26,6 @@ public class ScenarioGenerator
         }
 
         _buffer = new List<string>();
-        AddScenariosFileStart();
     }
 
     public void AddScenariosFileStart()

src/Test/Perf/util/tools_util.csx

@@ -31,6 +31,7 @@ bool ConvertConsumptionToCsv(string source, string destination, string requiredM
                     if (xmlReader.Name.Equals("ScenarioResult"))
                     {
                         currentScenarioName = xmlReader.GetAttribute("Name");
+                        currentScenarioName = new string(currentScenarioName.TakeWhile(c => !Char.IsDigit(c)).ToArray());
 
                         // These are not test results
                         if (string.Equals(currentScenarioName, "..TestDiagnostics.."))
@@ -132,16 +133,26 @@ void UploadTraces(string sourceFolderPath, string destinationFolderPath)
     Log("Uploading traces");
     if (Directory.Exists(sourceFolderPath))
     {
-        // Get the latest written databackup
-        var directoryToUpload = new DirectoryInfo(sourceFolderPath).GetDirectories("DataBackup*").OrderByDescending(d=>d.LastWriteTimeUtc).FirstOrDefault();
-        if (directoryToUpload == null)
+        var directoriesToUpload = new DirectoryInfo(sourceFolderPath).GetDirectories("DataBackup*");
+        if (directoriesToUpload.Count() == 0)
         {
             Log($"There are no trace directory starting with DataBackup in {sourceFolderPath}");
             return;
         }
 
-        var destination = Path.Combine(destinationFolderPath, directoryToUpload.Name);
-        CopyDirectory(directoryToUpload.FullName, destination);
+        var perfResultDestinationFolderName = string.Format("PerfResults-{0:yyyy-MM-dd_hh-mm-ss-tt}", DateTime.Now);
+        
+       var destination = Path.Combine(destinationFolderPath, perfResultDestinationFolderName);
+        foreach (var directoryToUpload in directoriesToUpload)
+        {
+            var destinationDataBackupDirectory = Path.Combine(destination, directoryToUpload.Name);
+            if (Directory.Exists(destinationDataBackupDirectory))
+            {
+                Directory.CreateDirectory(destinationDataBackupDirectory);
+            }
+            
+            CopyDirectory(directoryToUpload.FullName, destinationDataBackupDirectory);
+        }
 
         foreach(var file in new DirectoryInfo(sourceFolderPath).GetFiles().Where(f => f.Name.StartsWith("ConsumptionTemp", StringComparison.OrdinalIgnoreCase) || f.Name.StartsWith("Roslyn-", StringComparison.OrdinalIgnoreCase)))
         {

src/Test/Perf/util/trace_manager_util.csx

@@ -7,8 +7,9 @@ using System.IO;
 
 interface ITraceManager
 {
-    int Iterations { get; }
+    bool HasWarmUpIteration { get; }
 
+    void Initialize();
     void Cleanup();
     void EndEvent();
     void EndScenario();
@@ -18,24 +19,25 @@ interface ITraceManager
     void Start();
     void StartEvent();
     void StartScenario(string scenarioName, string processName);
+    void StartScenarios();
     void Stop();
     void WriteScenariosFileToDisk();
 }
 
 class TraceManagerFactory
 {
-    public static ITraceManager GetTraceManager(int iterations = 1)
+    public static ITraceManager GetTraceManager()
     {
         var directoryInfo = new RelativeDirectory();
         var cpcFullPath = Path.Combine(directoryInfo.CPCDirectoryPath, "CPC.exe");
         var scenarioPath = directoryInfo.CPCDirectoryPath;
         if (File.Exists(cpcFullPath))
         {
-            return new TraceManager(iterations, cpcFullPath, scenarioPath);
+            return new TraceManager(cpcFullPath, scenarioPath);
         }
         else
         {
-            return new NoOpTraceManager(iterations);
+            return new NoOpTraceManager();
         }
     }
 }
@@ -44,18 +46,20 @@ class TraceManagerFactory
 /// All operations are NoOp
 class NoOpTraceManager : ITraceManager
 {
-    private readonly int _iterations;
-    public NoOpTraceManager(int iterations)
+    public NoOpTraceManager()
     {
-        _iterations = iterations;
     }
 
-    public int Iterations
+    public bool HasWarmUpIteration
     {
         get
         {
-            return _iterations;
-        }
+            return false;  
+        }    
+    }
+    
+    public void Initialize()
+    {
     }
 
     public void Cleanup()
@@ -90,6 +94,10 @@ class NoOpTraceManager : ITraceManager
     {
     }
 
+    public void StartScenarios()
+    {
+    }
+
     public void StartScenario(string scenarioName, string processName)
     {
     }
@@ -106,7 +114,6 @@ class NoOpTraceManager : ITraceManager
 class TraceManager: ITraceManager
 {
     private readonly ScenarioGenerator _scenarioGenerator;
-    private readonly int _iterations;
     private readonly string _cpcPath;
     
     private RelativeDirectory _directoryInfo = new RelativeDirectory();
@@ -115,20 +122,37 @@ class TraceManager: ITraceManager
     private int _stopEventAbsoluteInstance = 1;
 
     public TraceManager(
-        int iterations,
         string cpcPath,
         string scenarioPath): base()
     {
-        _iterations = iterations;
         _cpcPath = cpcPath;
         _scenarioGenerator = new ScenarioGenerator(scenarioPath);
     }
 
-    public int Iterations
+    public bool HasWarmUpIteration
     {
         get
         {
-            return _iterations;
+            return true;
+        }    
+    }
+    
+    // Cleanup the results directory and files before every run
+    public void Initialize()
+    {
+        var consumptionTempResultsPath = Path.Combine(_directoryInfo.CPCDirectoryPath, "ConsumptionTempResults.xml");
+        if (File.Exists(consumptionTempResultsPath))
+        {
+            File.Delete(consumptionTempResultsPath);
+        }
+        
+        if (Directory.Exists(_directoryInfo.CPCDirectoryPath))
+        {
+            var databackDirectories = Directory.GetDirectories(_directoryInfo.CPCDirectoryPath, "DataBackup*", SearchOption.AllDirectories);
+            foreach (var databackDirectory in databackDirectories)
+            {
+                Directory.Delete(databackDirectory, true);
+            }
         }
     }
 
@@ -145,7 +169,7 @@ class TraceManager: ITraceManager
     public void Stop()
     {
         var scenariosXmlPath = Path.Combine(_directoryInfo.CPCDirectoryPath, "scenarios.xml");
-        var consumptionTempResultsPath = Path.Combine(_directoryInfo.CPCDirectoryPath, "ConsumptionTempResultsPath.xml");
+        var consumptionTempResultsPath = Path.Combine(_directoryInfo.CPCDirectoryPath, "ConsumptionTempResults.xml");
         ShellOutVital(_cpcPath, $"/Stop /DisableArchive /ScenarioPath=\"{scenariosXmlPath}\" /ConsumptionTempResultsPath=\"{consumptionTempResultsPath}\"");
     }
 
@@ -153,6 +177,11 @@ class TraceManager: ITraceManager
     {
         ShellOutVital(_cpcPath, "/Cleanup /DisableArchive");
     }
+    
+    public void StartScenarios()
+    {
+        _scenarioGenerator.AddScenariosFileStart();
+    }
 
     public void StartScenario(string scenarioName, string processName)
     {