From eb62d044d20291f0305faec1c69f71d05570ebbd Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Mon, 12 Sep 2016 22:15:38 -0700 Subject: [PATCH] Updating netci.groovy and RunTests.csproj to support the open integration tests. --- BuildAndTest.proj | 13 +++++++++-- cibuild.cmd | 3 ++- netci.groovy | 23 ++++++++++++++++++- src/Tools/Source/RunTests/ITestExecutor.cs | 7 ++++-- src/Tools/Source/RunTests/Options.cs | 10 ++++++++ src/Tools/Source/RunTests/Program.cs | 17 +++++++------- src/Tools/Source/RunTests/TestRunner.cs | 5 ++-- .../CSharp/Impl/CSharpVisualStudio.csproj | 2 +- .../CSharp/Repl/CSharpVisualStudioRepl.csproj | 2 +- .../Core/Def/ServicesVisualStudio.csproj | 2 +- .../VisualBasic/Impl/BasicVisualStudio.vbproj | 2 +- .../Repl/BasicVisualStudioRepl.vbproj | 2 +- 12 files changed, 67 insertions(+), 21 deletions(-) diff --git a/BuildAndTest.proj b/BuildAndTest.proj index eca3b81ee8f..0ab8fe415e5 100644 --- a/BuildAndTest.proj +++ b/BuildAndTest.proj @@ -13,17 +13,26 @@ Debug $(RunTestArgs) -xml $(RunTestArgs) -test64 + $(RunTestArgs) -testVSI $(RunTestArgs) -trait:$(Trait) $(RunTestArgs) -notrait:$(NoTrait) - *.UnitTests*.dll + *.UnitTests*.dll + *.IntegrationTests*.dll Binaries\$(Configuration) $(UserProfile)\.nuget\packages $(OutputDirectory)\CoreClrTest RestorePackages=false; TreatWarningsAsErrors=true; + + + $(MSBuildCommonProperties); DeployExtension=false; + + $(MSBuildCommonProperties); + DeployExtension=true; + @@ -102,7 +111,7 @@ $(NuGetPackageRoot)\xunit.runner.console\$(xunitrunnerconsoleVersion)\tools $(RunTestArgs) @(TestAssemblies, ' ') - + diff --git a/cibuild.cmd b/cibuild.cmd index 44856aed864..1efe5ab3657 100644 --- a/cibuild.cmd +++ b/cibuild.cmd @@ -20,6 +20,7 @@ if /I "%1" == "/test64" set Test64=true&&shift&& goto :ParseArguments if /I "%1" == "/testDeterminism" set TestDeterminism=true&&shift&& goto :ParseArguments if /I "%1" == "/testPerfCorrectness" set TestPerfCorrectness=true&&shift&& goto :ParseArguments if /I "%1" == "/testPerfRun" set TestPerfRun=true&&shift&& goto :ParseArguments +if /I "%1" == "/testVSI" set TestVSI=true&&shift&& goto :ParseArguments REM /buildTimeLimit is the time limit, measured in minutes, for the Jenkins job that runs REM the build. The Jenkins script netci.groovy passes the time limit to this script. @@ -90,7 +91,7 @@ if defined TestPerfRun ( exit /b 0 ) -msbuild %MSBuildAdditionalCommandLineArgs% /p:BootstrapBuildPath="%bindir%\Bootstrap" BuildAndTest.proj /p:Configuration=%BuildConfiguration% /p:Test64=%Test64% /p:RunProcessWatchdog=%RunProcessWatchdog% /p:BuildStartTime=%BuildStartTime% /p:"ProcDumpExe=%ProcDumpExe%" /p:BuildTimeLimit=%BuildTimeLimit% /p:PathMap="%RoslynRoot%=q:\roslyn" /p:Feature=pdb-path-determinism /fileloggerparameters:LogFile="%bindir%\Build.log";verbosity=diagnostic || goto :BuildFailed +msbuild %MSBuildAdditionalCommandLineArgs% /p:BootstrapBuildPath="%bindir%\Bootstrap" BuildAndTest.proj /p:Configuration=%BuildConfiguration% /p:Test64=%Test64% /p:TestVSI=%TestVSI% /p:RunProcessWatchdog=%RunProcessWatchdog% /p:BuildStartTime=%BuildStartTime% /p:"ProcDumpExe=%ProcDumpExe%" /p:BuildTimeLimit=%BuildTimeLimit% /p:PathMap="%RoslynRoot%=q:\roslyn" /p:Feature=pdb-path-determinism /fileloggerparameters:LogFile="%bindir%\Build.log";verbosity=diagnostic || goto :BuildFailed powershell -noprofile -executionPolicy RemoteSigned -file "%RoslynRoot%\build\scripts\check-msbuild.ps1" "%bindir%\Build.log" || goto :BuildFailed call :TerminateBuildProcesses diff --git a/netci.groovy b/netci.groovy index 4eed6e43e4a..f44a64724f8 100644 --- a/netci.groovy +++ b/netci.groovy @@ -15,7 +15,7 @@ static void addEmailPublisher(def myJob) { myJob.with { publishers { extendedEmail('mlinfraswat@microsoft.com', '$DEFAULT_SUBJECT', '$DEFAULT_CONTENT') { - // trigger(trigger name, subject, body, recipient list, send to developers, send to requester, include culprits, send to recipient list) + // trigger(trigger name, subject, body, recipient list, send to developers, send to requester, include culprits, send to recipient list) trigger('Aborted', '$PROJECT_DEFAULT_SUBJECT', '$PROJECT_DEFAULT_CONTENT', null, false, false, false, true) trigger('Failure', '$PROJECT_DEFAULT_SUBJECT', '$PROJECT_DEFAULT_CONTENT', null, false, false, false, true) } @@ -198,6 +198,27 @@ commitPullList.each { isPr -> addRoslynJob(myJob, jobName, branchName, isPr, triggerPhraseExtra, triggerPhraseOnly) } +// Open Integration Tests +commitPullList.each { isPr -> + def jobName = Utilities.getFullJobName(projectName, "open-vsi", isPr) + def myJob = job(jobName) { + description('open integration tests') + label('auto-win2012-20160912') + steps { + batchFile("""set TEMP=%WORKSPACE%\\Binaries\\Temp +mkdir %TEMP% +set TMP=%TEMP% +set VS150COMNTOOLS=%ProgramFiles(x86)%\\Microsoft Visual Studio\\VS15Preview\\Common7\\Tools\\ +.\\cibuild.cmd /debug /testVSI""") + } + } + + def triggerPhraseOnly = true + def triggerPhraseExtra = "open-vsi" + Utilities.setMachineAffinity(myJob, 'Windows_NT', 'latest-or-auto-dev15-preview4') + addRoslynJob(myJob, jobName, branchName, isPr, triggerPhraseExtra, triggerPhraseOnly) +} + JobReport.Report.generateJobReport(out) // Make the call to generate the help job diff --git a/src/Tools/Source/RunTests/ITestExecutor.cs b/src/Tools/Source/RunTests/ITestExecutor.cs index da6a6944c8b..cfa58ab614c 100644 --- a/src/Tools/Source/RunTests/ITestExecutor.cs +++ b/src/Tools/Source/RunTests/ITestExecutor.cs @@ -16,13 +16,16 @@ internal struct TestExecutionOptions internal bool UseHtml { get; } internal bool Test64 { get; } - internal TestExecutionOptions(string xunitPath, string trait, string noTrait, bool useHtml, bool test64) + internal bool TestVSI { get; } + + internal TestExecutionOptions(string xunitPath, string trait, string noTrait, bool useHtml, bool test64, bool testVSI) { XunitPath = xunitPath; Trait = trait; NoTrait = noTrait; UseHtml = useHtml; Test64 = test64; + TestVSI = testVSI; } } @@ -40,7 +43,7 @@ internal struct TestResult internal bool IsResultFromCache { get; } /// - /// Path to the results file. Can be null in the case xunit error'd and did not create one. + /// Path to the results file. Can be null in the case xunit error'd and did not create one. /// internal string ResultsFilePath { get; } diff --git a/src/Tools/Source/RunTests/Options.cs b/src/Tools/Source/RunTests/Options.cs index b94a133f01d..d63b9b2f14c 100644 --- a/src/Tools/Source/RunTests/Options.cs +++ b/src/Tools/Source/RunTests/Options.cs @@ -27,6 +27,11 @@ internal class Options /// public bool Test64 { get; set; } + /// + /// Use the open integration test runner. + /// + public bool TestVSI { get; set; } + /// /// Allow the caching of test results. /// @@ -77,6 +82,11 @@ internal static Options Parse(string[] args) opt.Test64 = true; index++; } + else if (comp.Equals(current, "-testVSI")) + { + opt.TestVSI = true; + index++; + } else if (comp.Equals(current, "-xml")) { opt.UseHtml = false; diff --git a/src/Tools/Source/RunTests/Program.cs b/src/Tools/Source/RunTests/Program.cs index fc552736ea5..bcb4681df12 100644 --- a/src/Tools/Source/RunTests/Program.cs +++ b/src/Tools/Source/RunTests/Program.cs @@ -43,7 +43,7 @@ internal static int Main(string[] args) private static async Task RunCore(Options options, CancellationToken cancellationToken) { if (!CheckAssemblyList(options)) - { + { return ExitFailure; } @@ -80,7 +80,7 @@ private static async Task RunCore(Options options, CancellationToken cancel } /// - /// Quick sanity check to look over the set of assemblies to make sure they are valid and something was + /// Quick sanity check to look over the set of assemblies to make sure they are valid and something was /// specified. /// private static bool CheckAssemblyList(Options options) @@ -118,7 +118,7 @@ private static List GetAssemblyList(Options options) { var name = Path.GetFileName(assemblyPath); - // As a starting point we will just schedule the items we know to be a performance + // As a starting point we will just schedule the items we know to be a performance // bottleneck. Can adjust as we get real data. if (name == "Roslyn.Compilers.CSharp.Emit.UnitTests.dll" || name == "Roslyn.Services.Editor.UnitTests.dll" || @@ -170,9 +170,9 @@ private static bool CanUseWebStorage() { // The web caching layer is still being worked on. For now want to limit it to Roslyn developers // and Jenkins runs by default until we work on this a bit more. Anyone reading this who wants - // to try it out should feel free to opt into this. - return - StringComparer.OrdinalIgnoreCase.Equals("REDMOND", Environment.UserDomainName) || + // to try it out should feel free to opt into this. + return + StringComparer.OrdinalIgnoreCase.Equals("REDMOND", Environment.UserDomainName) || Constants.IsJenkinsRun; } @@ -183,7 +183,8 @@ private static ITestExecutor CreateTestExecutor(Options options) trait: options.Trait, noTrait: options.NoTrait, useHtml: options.UseHtml, - test64: options.Test64); + test64: options.Test64, + testVSI: options.TestVSI); var processTestExecutor = new ProcessTestExecutor(testExecutionOptions); if (!options.UseCachedResults) { @@ -192,7 +193,7 @@ private static ITestExecutor CreateTestExecutor(Options options) // The web caching layer is still being worked on. For now want to limit it to Roslyn developers // and Jenkins runs by default until we work on this a bit more. Anyone reading this who wants - // to try it out should feel free to opt into this. + // to try it out should feel free to opt into this. IDataStorage dataStorage = new LocalDataStorage(); if (CanUseWebStorage()) { diff --git a/src/Tools/Source/RunTests/TestRunner.cs b/src/Tools/Source/RunTests/TestRunner.cs index 8c1087709d4..a584b8ad1b8 100644 --- a/src/Tools/Source/RunTests/TestRunner.cs +++ b/src/Tools/Source/RunTests/TestRunner.cs @@ -40,7 +40,8 @@ internal TestRunner(Options options, ITestExecutor testExecutor) internal async Task RunAllAsync(IEnumerable assemblyInfoList, CancellationToken cancellationToken) { - var max = (int)(Environment.ProcessorCount * 1.5); + // Use 1.5 times the number of processors for unit tests, but only 1 processor for the open integration tests + var max = (_options.TestVSI) ? 1 : (int)(Environment.ProcessorCount * 1.5); var allPassed = true; var cacheCount = 0; var waiting = new Stack(assemblyInfoList); @@ -131,7 +132,7 @@ private void PrintFailedTestResult(TestResult testResult) Console.WriteLine("Errors {0}: ", testResult.AssemblyName); Console.WriteLine(testResult.ErrorOutput); - // TODO: Put this in the log and take it off the console output to keep it simple? + // TODO: Put this in the log and take it off the console output to keep it simple? Console.WriteLine($"Command: {testResult.CommandLine}"); Console.WriteLine($"xUnit output log: {outputLogPath}"); diff --git a/src/VisualStudio/CSharp/Impl/CSharpVisualStudio.csproj b/src/VisualStudio/CSharp/Impl/CSharpVisualStudio.csproj index ae83fb477ff..c8be592c712 100644 --- a/src/VisualStudio/CSharp/Impl/CSharpVisualStudio.csproj +++ b/src/VisualStudio/CSharp/Impl/CSharpVisualStudio.csproj @@ -1,5 +1,5 @@  - + diff --git a/src/VisualStudio/CSharp/Repl/CSharpVisualStudioRepl.csproj b/src/VisualStudio/CSharp/Repl/CSharpVisualStudioRepl.csproj index 2af87251962..f5db7412e87 100644 --- a/src/VisualStudio/CSharp/Repl/CSharpVisualStudioRepl.csproj +++ b/src/VisualStudio/CSharp/Repl/CSharpVisualStudioRepl.csproj @@ -1,5 +1,5 @@  - + CSharp diff --git a/src/VisualStudio/Core/Def/ServicesVisualStudio.csproj b/src/VisualStudio/Core/Def/ServicesVisualStudio.csproj index ca520c15371..4edd368da0f 100644 --- a/src/VisualStudio/Core/Def/ServicesVisualStudio.csproj +++ b/src/VisualStudio/Core/Def/ServicesVisualStudio.csproj @@ -1,5 +1,5 @@  - + CSharp diff --git a/src/VisualStudio/VisualBasic/Impl/BasicVisualStudio.vbproj b/src/VisualStudio/VisualBasic/Impl/BasicVisualStudio.vbproj index 07e9d061ac3..e00d14cab9b 100644 --- a/src/VisualStudio/VisualBasic/Impl/BasicVisualStudio.vbproj +++ b/src/VisualStudio/VisualBasic/Impl/BasicVisualStudio.vbproj @@ -1,5 +1,5 @@  - + VB diff --git a/src/VisualStudio/VisualBasic/Repl/BasicVisualStudioRepl.vbproj b/src/VisualStudio/VisualBasic/Repl/BasicVisualStudioRepl.vbproj index 0d28298e3f3..55bc02b598e 100644 --- a/src/VisualStudio/VisualBasic/Repl/BasicVisualStudioRepl.vbproj +++ b/src/VisualStudio/VisualBasic/Repl/BasicVisualStudioRepl.vbproj @@ -1,5 +1,5 @@  - + Debug -- GitLab