Updating netci.groovy and RunTests.csproj to support the open integration tests.

BuildAndTest.proj

@@ -13,17 +13,26 @@
     <Configuration Condition="'$(Configuration)' == ''">Debug</Configuration>
     <RunTestArgs Condition="'$(ManualTest)' == ''">$(RunTestArgs) -xml</RunTestArgs>
     <RunTestArgs Condition="'$(Test64)' == 'true'">$(RunTestArgs) -test64</RunTestArgs>
+    <RunTestArgs Condition="'$(TestVSI)' == 'true'">$(RunTestArgs) -testVSI</RunTestArgs>
     <RunTestArgs Condition="'$(Trait)' != ''">$(RunTestArgs) -trait:$(Trait)</RunTestArgs>
     <RunTestArgs Condition="'$(NoTrait)' != ''">$(RunTestArgs) -notrait:$(NoTrait)</RunTestArgs>
-    <IncludePattern Condition="'$(IncludePattern)' == ''">*.UnitTests*.dll</IncludePattern>
+    <IncludePattern Condition="'$(IncludePattern)' == '' AND '$(TestVSI)' != 'true'">*.UnitTests*.dll</IncludePattern>
+    <IncludePattern Condition="'$(IncludePattern)' == '' AND '$(TestVSI)' == 'true'">*.IntegrationTests*.dll</IncludePattern>
     <OutputDirectory>Binaries\$(Configuration)</OutputDirectory>
     <NuGetPackageRoot>$(UserProfile)\.nuget\packages</NuGetPackageRoot>
     <CoreClrTestDirectory>$(OutputDirectory)\CoreClrTest</CoreClrTestDirectory>
     <MSBuildCommonProperties>
       RestorePackages=false;
       TreatWarningsAsErrors=true;
+    </MSBuildCommonProperties>
+    <MSBuildCommonProperties Condition="'$(TestVSI)' != 'true'">
+      $(MSBuildCommonProperties);
       DeployExtension=false;
     </MSBuildCommonProperties>
+    <MSBuildCommonProperties Condition="'$(TestVSI)' == 'true'">
+      $(MSBuildCommonProperties);
+      DeployExtension=true;
+    </MSBuildCommonProperties>
   </PropertyGroup>
 
   <Target Name="Build">
@@ -102,7 +111,7 @@
       <RunTestsArgs>$(NuGetPackageRoot)\xunit.runner.console\$(xunitrunnerconsoleVersion)\tools $(RunTestArgs) @(TestAssemblies, ' ')</RunTestsArgs>
     </PropertyGroup>
     
-    <Exec Condition="'$(RunProcessWatchdog)' != 'true'" Command="&quot;$(CoreRunExe)&quot; $(CoreRunArgs)" />
+    <Exec Condition="'$(RunProcessWatchdog)' != 'true' AND '$(TestVSI)' != 'true'" Command="&quot;$(CoreRunExe)&quot; $(CoreRunArgs)" />
 
     <Exec Condition="'$(RunProcessWatchdog)' != 'true'" Command="&quot;$(RunTestsExe)&quot; $(RunTestsArgs)" />
 

cibuild.cmd

@@ -20,6 +20,7 @@ if /I "%1" == "/test64" set Test64=true&&shift&& goto :ParseArguments
 if /I "%1" == "/testDeterminism" set TestDeterminism=true&&shift&& goto :ParseArguments
 if /I "%1" == "/testPerfCorrectness" set TestPerfCorrectness=true&&shift&& goto :ParseArguments
 if /I "%1" == "/testPerfRun" set TestPerfRun=true&&shift&& goto :ParseArguments
+if /I "%1" == "/testVSI" set TestVSI=true&&shift&& goto :ParseArguments
 
 REM /buildTimeLimit is the time limit, measured in minutes, for the Jenkins job that runs
 REM the build. The Jenkins script netci.groovy passes the time limit to this script.
@@ -90,7 +91,7 @@ if defined TestPerfRun (
     exit /b 0
 )
 
-msbuild %MSBuildAdditionalCommandLineArgs% /p:BootstrapBuildPath="%bindir%\Bootstrap" BuildAndTest.proj /p:Configuration=%BuildConfiguration% /p:Test64=%Test64% /p:RunProcessWatchdog=%RunProcessWatchdog% /p:BuildStartTime=%BuildStartTime% /p:"ProcDumpExe=%ProcDumpExe%" /p:BuildTimeLimit=%BuildTimeLimit% /p:PathMap="%RoslynRoot%=q:\roslyn" /p:Feature=pdb-path-determinism /fileloggerparameters:LogFile="%bindir%\Build.log";verbosity=diagnostic || goto :BuildFailed
+msbuild %MSBuildAdditionalCommandLineArgs% /p:BootstrapBuildPath="%bindir%\Bootstrap" BuildAndTest.proj /p:Configuration=%BuildConfiguration% /p:Test64=%Test64% /p:TestVSI=%TestVSI% /p:RunProcessWatchdog=%RunProcessWatchdog% /p:BuildStartTime=%BuildStartTime% /p:"ProcDumpExe=%ProcDumpExe%" /p:BuildTimeLimit=%BuildTimeLimit% /p:PathMap="%RoslynRoot%=q:\roslyn" /p:Feature=pdb-path-determinism /fileloggerparameters:LogFile="%bindir%\Build.log";verbosity=diagnostic || goto :BuildFailed
 powershell -noprofile -executionPolicy RemoteSigned -file "%RoslynRoot%\build\scripts\check-msbuild.ps1" "%bindir%\Build.log" || goto :BuildFailed
 
 call :TerminateBuildProcesses

netci.groovy

@@ -15,7 +15,7 @@ static void addEmailPublisher(def myJob) {
   myJob.with {
     publishers {
       extendedEmail('mlinfraswat@microsoft.com', '$DEFAULT_SUBJECT', '$DEFAULT_CONTENT') {
-	// trigger(trigger name, subject, body, recipient list, send to developers, send to requester, include culprits, send to recipient list)
+        // trigger(trigger name, subject, body, recipient list, send to developers, send to requester, include culprits, send to recipient list)
         trigger('Aborted', '$PROJECT_DEFAULT_SUBJECT', '$PROJECT_DEFAULT_CONTENT', null, false, false, false, true)
         trigger('Failure', '$PROJECT_DEFAULT_SUBJECT', '$PROJECT_DEFAULT_CONTENT', null, false, false, false, true)
       }
@@ -198,6 +198,27 @@ commitPullList.each { isPr ->
   addRoslynJob(myJob, jobName, branchName, isPr, triggerPhraseExtra, triggerPhraseOnly)
 }
 
+// Open Integration Tests
+commitPullList.each { isPr ->
+  def jobName = Utilities.getFullJobName(projectName, "open-vsi", isPr)
+  def myJob = job(jobName) {
+    description('open integration tests')
+    label('auto-win2012-20160912')
+    steps {
+      batchFile("""set TEMP=%WORKSPACE%\\Binaries\\Temp
+mkdir %TEMP%
+set TMP=%TEMP%
+set VS150COMNTOOLS=%ProgramFiles(x86)%\\Microsoft Visual Studio\\VS15Preview\\Common7\\Tools\\
+.\\cibuild.cmd /debug /testVSI""")
+    }
+  }
+
+  def triggerPhraseOnly = true
+  def triggerPhraseExtra = "open-vsi"
+  Utilities.setMachineAffinity(myJob, 'Windows_NT', 'latest-or-auto-dev15-preview4')
+  addRoslynJob(myJob, jobName, branchName, isPr, triggerPhraseExtra, triggerPhraseOnly)
+}
+
 JobReport.Report.generateJobReport(out)
 
 // Make the call to generate the help job

src/Tools/Source/RunTests/ITestExecutor.cs

@@ -16,13 +16,16 @@ internal struct TestExecutionOptions
         internal bool UseHtml { get; }
         internal bool Test64 { get; }
 
-        internal TestExecutionOptions(string xunitPath, string trait, string noTrait, bool useHtml, bool test64)
+        internal bool TestVSI { get; }
+
+        internal TestExecutionOptions(string xunitPath, string trait, string noTrait, bool useHtml, bool test64, bool testVSI)
         {
             XunitPath = xunitPath;
             Trait = trait;
             NoTrait = noTrait;
             UseHtml = useHtml;
             Test64 = test64;
+            TestVSI = testVSI;
         }
     }
 
@@ -40,7 +43,7 @@ internal struct TestResult
         internal bool IsResultFromCache { get; }
 
         /// <summary>
-        /// Path to the results file.  Can be null in the case xunit error'd and did not create one. 
+        /// Path to the results file.  Can be null in the case xunit error'd and did not create one.
         /// </summary>
         internal string ResultsFilePath { get; }
 

src/Tools/Source/RunTests/Options.cs

@@ -27,6 +27,11 @@ internal class Options
         /// </summary>
         public bool Test64 { get; set; }
 
+        /// <summary>
+        /// Use the open integration test runner.
+        /// </summary>
+        public bool TestVSI { get; set; }
+
         /// <summary>
         /// Allow the caching of test results.
         /// </summary>
@@ -77,6 +82,11 @@ internal static Options Parse(string[] args)
                     opt.Test64 = true;
                     index++;
                 }
+                else if (comp.Equals(current, "-testVSI"))
+                {
+                    opt.TestVSI = true;
+                    index++;
+                }
                 else if (comp.Equals(current, "-xml"))
                 {
                     opt.UseHtml = false;

src/Tools/Source/RunTests/Program.cs

@@ -43,7 +43,7 @@ internal static int Main(string[] args)
         private static async Task<int> RunCore(Options options, CancellationToken cancellationToken)
         {
             if (!CheckAssemblyList(options))
-            { 
+            {
                 return ExitFailure;
             }
 
@@ -80,7 +80,7 @@ private static async Task<int> RunCore(Options options, CancellationToken cancel
         }
 
         /// <summary>
-        /// Quick sanity check to look over the set of assemblies to make sure they are valid and something was 
+        /// Quick sanity check to look over the set of assemblies to make sure they are valid and something was
         /// specified.
         /// </summary>
         private static bool CheckAssemblyList(Options options)
@@ -118,7 +118,7 @@ private static List<AssemblyInfo> GetAssemblyList(Options options)
             {
                 var name = Path.GetFileName(assemblyPath);
 
-                // As a starting point we will just schedule the items we know to be a performance 
+                // As a starting point we will just schedule the items we know to be a performance
                 // bottleneck.  Can adjust as we get real data.
                 if (name == "Roslyn.Compilers.CSharp.Emit.UnitTests.dll" ||
                     name == "Roslyn.Services.Editor.UnitTests.dll" ||
@@ -170,9 +170,9 @@ private static bool CanUseWebStorage()
         {
             // The web caching layer is still being worked on.  For now want to limit it to Roslyn developers
             // and Jenkins runs by default until we work on this a bit more.  Anyone reading this who wants
-            // to try it out should feel free to opt into this. 
-            return 
-                StringComparer.OrdinalIgnoreCase.Equals("REDMOND", Environment.UserDomainName) || 
+            // to try it out should feel free to opt into this.
+            return
+                StringComparer.OrdinalIgnoreCase.Equals("REDMOND", Environment.UserDomainName) ||
                 Constants.IsJenkinsRun;
         }
 
@@ -183,7 +183,8 @@ private static ITestExecutor CreateTestExecutor(Options options)
                 trait: options.Trait,
                 noTrait: options.NoTrait,
                 useHtml: options.UseHtml,
-                test64: options.Test64);
+                test64: options.Test64,
+                testVSI: options.TestVSI);
             var processTestExecutor = new ProcessTestExecutor(testExecutionOptions);
             if (!options.UseCachedResults)
             {
@@ -192,7 +193,7 @@ private static ITestExecutor CreateTestExecutor(Options options)
 
             // The web caching layer is still being worked on.  For now want to limit it to Roslyn developers
             // and Jenkins runs by default until we work on this a bit more.  Anyone reading this who wants
-            // to try it out should feel free to opt into this. 
+            // to try it out should feel free to opt into this.
             IDataStorage dataStorage = new LocalDataStorage();
             if (CanUseWebStorage())
             {

src/Tools/Source/RunTests/TestRunner.cs

@@ -40,7 +40,8 @@ internal TestRunner(Options options, ITestExecutor testExecutor)
 
         internal async Task<RunAllResult> RunAllAsync(IEnumerable<AssemblyInfo> assemblyInfoList, CancellationToken cancellationToken)
         {
-            var max = (int)(Environment.ProcessorCount * 1.5);
+            // Use 1.5 times the number of processors for unit tests, but only 1 processor for the open integration tests
+            var max = (_options.TestVSI) ? 1 : (int)(Environment.ProcessorCount * 1.5);
             var allPassed = true;
             var cacheCount = 0;
             var waiting = new Stack<AssemblyInfo>(assemblyInfoList);
@@ -131,7 +132,7 @@ private void PrintFailedTestResult(TestResult testResult)
             Console.WriteLine("Errors {0}: ", testResult.AssemblyName);
             Console.WriteLine(testResult.ErrorOutput);
 
-            // TODO: Put this in the log and take it off the console output to keep it simple? 
+            // TODO: Put this in the log and take it off the console output to keep it simple?
             Console.WriteLine($"Command: {testResult.CommandLine}");
             Console.WriteLine($"xUnit output log: {outputLogPath}");
 

src/VisualStudio/CSharp/Impl/CSharpVisualStudio.csproj

 <?xml version="1.0" encoding="utf-8"?>
-<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" TreatAsLocalProperty="DeployExtension">
   <PropertyGroup>
     <!-- Explicitly set the language before anything else is imported. That way VSL.Settings.targets sees the right value
          even in WPF's build of a temporary project, where it won't be able to determine it based on the file extension. -->

src/VisualStudio/CSharp/Repl/CSharpVisualStudioRepl.csproj

 <?xml version="1.0" encoding="utf-8"?>
-<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" TreatAsLocalProperty="DeployExtension">
   <PropertyGroup>
     <ProjectLanguage>CSharp</ProjectLanguage>
   </PropertyGroup>

src/VisualStudio/Core/Def/ServicesVisualStudio.csproj

 <?xml version="1.0" encoding="utf-8"?>
-<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" TreatAsLocalProperty="DeployExtension">
   <PropertyGroup>
     <ProjectLanguage>CSharp</ProjectLanguage>
   </PropertyGroup>

src/VisualStudio/VisualBasic/Impl/BasicVisualStudio.vbproj

 <?xml version="1.0" encoding="utf-8"?>
-<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" TreatAsLocalProperty="DeployExtension">
   <PropertyGroup>
     <ProjectLanguage>VB</ProjectLanguage>
   </PropertyGroup>

src/VisualStudio/VisualBasic/Repl/BasicVisualStudioRepl.vbproj

 <?xml version="1.0" encoding="utf-8"?>
-<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" TreatAsLocalProperty="DeployExtension">
   <Import Project="..\..\..\..\build\Targets\VSL.Settings.targets" />
   <PropertyGroup>
     <Configuration Condition="'$(Configuration)' == ''">Debug</Configuration>