Merge pull request #31372 from tmat/VSMSBuild

Move VS MSBuild location to Arcade tools script

.gitignore

@@ -16,6 +16,7 @@ x64/
 [Bb]in/
 [Oo]bj/
 .dotnet/
+.tools/
 
 # Enable "build/" folder in the NuGet Packages folder since NuGet packages use it for MSBuild targets
 !packages/*/build/

build/Targets/Tools.props

@@ -8,20 +8,5 @@
 
   <PropertyGroup>
     <dotnetRuntimeVersion>2.1.3</dotnetRuntimeVersion>
-
-    <!--
-        When changing this value ensure that you do not exceed the LKG used by source build. This is necessary
-        to ensure we maintain our ability to build from source. The LKG is stored here.
-
-        https://github.com/dotnet/source-build/blob/master/DotnetCLIVersion.txt  
-
-        Also when changing this also make sure to:
-
-        - Update links and version numbers in docs/contributing/Building%2C%20Debugging%2C%20and%20Testing%20on%20Windows.md
-        - Email the team to ask them to upgrade to the new SDK
-    -->
-    <dotnetSdkVersion>2.1.401</dotnetSdkVersion>
-    <monoVersion>5.8.0.88</monoVersion>
-    <vsMinimumVersion>15.7</vsMinimumVersion>
   </PropertyGroup>
 </Project>

build/scripts/build-utils.ps1

@@ -31,6 +31,8 @@ function Exec-Block([scriptblock]$cmd) {
 # This will exec a process using the console and return it's exit code. This will not 
 # throw when the process fails.
 function Exec-Process([string]$command, [string]$commandArgs) {
+    Write-Host $command $commandArgs
+
     $startInfo = New-Object System.Diagnostics.ProcessStartInfo
     $startInfo.FileName = $command
     $startInfo.Arguments = $commandArgs
@@ -68,6 +70,8 @@ function Exec-CommandCore([string]$command, [string]$commandArgs, [switch]$useCo
         return
     }
 
+    Write-Host $command $commandArgs
+
     $startInfo = New-Object System.Diagnostics.ProcessStartInfo
     $startInfo.FileName = $command
     $startInfo.Arguments = $commandArgs
@@ -142,8 +146,7 @@ function Exec-Script([string]$script, [string]$scriptArgs = "") {
     Exec-Command "powershell" "-noprofile -executionPolicy RemoteSigned -file `"$script`" $scriptArgs"
 }
 
-# Ensure the proper SDK in installed in our %PATH%. This is how MSBuild locates the 
-# SDK. Returns the location to the dotnet exe
+# Ensure the proper .NET Core SDK is available. Returns the location to the dotnet.exe.
 function Ensure-DotnetSdk() {
   if (-not (Test-Path global:_dotNetExe)) {
     $global:_dotNetExe = Join-Path (InitializeDotNetCli -install:$true) "dotnet.exe"
@@ -151,57 +154,14 @@ function Ensure-DotnetSdk() {
 
   return $global:_dotNetExe
 }
-# Ensure a basic tool used for building our Repo is installed and 
-# return the path to it.
-function Ensure-BasicTool([string]$name, [string]$version = "") {
-    if ($version -eq "") { 
-        $version = Get-PackageVersion $name
-    }
-
-    $p = Join-Path (Get-PackagesDir) "$($name)\$($version)"
-    if (-not (Test-Path $p)) {
-        $toolsetProject = Join-Path $RepoRoot "build\ToolsetPackages\RoslynToolset.csproj"
-        Write-Host "Downloading $name"
-        Restore-Project $toolsetProject
-    }
-    
-    return $p
-}
-
-# Ensure that MSBuild is installed and return the path to the
-# executable to use.
-function Ensure-MSBuild([switch]$xcopy = $false) {
-    $both = Get-MSBuildKindAndDir -xcopy:$xcopy
-    $msbuildDir = $both[1]
-    switch ($both[0]) {
-        "xcopy" { break; }
-        "vscmd" { break; }
-        "vsinstall" { break; }
-        default {
-            throw "Unknown MSBuild installation type $($both[0])"
-        }
-    }
-
-    return Join-Path $msbuildDir "msbuild.exe"
-}
-
-# Returns the msbuild exe path and directory as a single return. This makes it easy 
-# to do one line MSBuild configuration in scripts
-#   $msbuild, $msbuildDir = Ensure-MSBuildAndDir
-function Ensure-MSBuildAndDir([string]$msbuildDir) {
-    if ($msbuildDir -eq "") {
-        $msbuild = Ensure-MSBuild
-        $msbuildDir = Split-Path -parent $msbuild
-    }
-    else {
-        $msbuild = Join-Path $msbuildDir "msbuild.exe"
-    }
 
-    return $msbuild, $msbuildDir
-}
+# Ensure the proper VS msbuild is available. Returns the locaqtion of the msbuild.exe.
+function Ensure-MSBuild() {
+  if (-not (Test-Path global:_msbuildExe)) {
+    $global:_msbuildExe = InitializeVisualStudioMSBuild
+  }
 
-function Create-Directory([string]$dir) {
-    New-Item $dir -ItemType Directory -ErrorAction SilentlyContinue | Out-Null
+  return $global:_msbuildExe
 }
 
 function Get-VersionCore([string]$name, [string]$versionFile) {
@@ -256,121 +216,6 @@ function Get-PackageDir([string]$name, [string]$version = "") {
     return $p
 }
 
-# The intent of this script is to locate and return the path to the MSBuild directory that
-# we should use for bulid operations.  The preference order for MSBuild to use is as 
-# follows:
-#
-#   1. MSBuild from an active VS command prompt
-#   2. MSBuild from a compatible VS installation
-#   3. MSBuild from the xcopy toolset 
-#
-# This function will return two values: the kind of MSBuild chosen and the MSBuild directory.
-function Get-MSBuildKindAndDir([switch]$xcopy = $false) {
-
-    if ($xcopy) { 
-        Write-Output "xcopy"
-        Write-Output (Get-MSBuildDirXCopy)
-        return
-    }
-
-    # MSBuild from an active VS command prompt. Use the MSBuild here so long as it's from a 
-    # compatible Visual Studio. If not though throw and error out. Given the number of 
-    # environment variable changes in a developer command prompt it's hard to make guarantees
-    # about subbing in a new MSBuild instance
-    if (${env:VSINSTALLDIR} -ne $null) {
-        $command = (Get-Command msbuild -ErrorAction SilentlyContinue)
-        if ((Test-SupportedVisualStudioVersion ${env:VSCMD_VER}) -and ($command -ne $null) ) {
-            $p = Split-Path -parent $command.Path
-            Write-Output "vscmd"
-            Write-Output $p
-            return
-        }
-        else {
-            $vsMinimumVersion = Get-ToolVersion "vsMinimum"
-            throw "Developer Command Prompt for VS $(${env:VSCMD_VER}) is not recent enough. Please upgrade to {$vsMinimumVersion} or build from a normal CMD window"
-        }
-    }
-
-    # Look for a valid VS installation
-    try {
-        $p = Get-VisualStudioDir
-        $p = Join-Path $p "MSBuild\15.0\Bin"
-        Write-Output "vsinstall"
-        Write-Output $p
-        return
-    }
-    catch { 
-        # Failures are expected here when no VS installation is present on the 
-        # machine.
-    }
-
-    Write-Output "xcopy"
-    Write-Output (Get-MSBuildDirXCopy)
-    return
-}
-
-# Locate the xcopy version of MSBuild
-function Get-MSBuildDirXCopy() {
-    $p = Ensure-BasicTool "RoslynTools.MSBuild"
-    $p = Join-Path $p "tools\MSBuild\15.0\Bin"
-    return $p
-}
-
-function Get-MSBuildDir([switch]$xcopy = $false) {
-    $both = Get-MSBuildKindAndDir -xcopy:$xcopy
-    return $both[1]
-}
-
-
-# Dose this version of Visual Studio meet our minimum requirements for building.
-function Test-SupportedVisualStudioVersion([string]$version) { 
-    # This regex allows us to strip off any pre-release info that gets attached 
-    # to the version string. VS uses NuGet style pre-release by suffing version
-    # with -<pre-release info>
-    if (-not ($version -match "^([\d.]+)(\+|-)?.*$")) { 
-        return $false
-    }
-
-    $vsMinimumVersion = Get-ToolVersion "vsMinimum"
-    $V = New-Object System.Version $matches[1]
-    $min = New-Object System.Version $vsMinimumVersion
-    return $v -ge $min;
-}
-
-# Get the directory and instance ID of the first Visual Studio version which 
-# meets our minimal requirements for the Roslyn repo.
-function Get-VisualStudioDirAndId() {
-    $vswhere = Join-Path (Ensure-BasicTool "vswhere") "tools\vswhere.exe"
-    $output = Exec-Command $vswhere "-prerelease -requires Microsoft.Component.MSBuild -format json" | Out-String
-    $j = ConvertFrom-Json $output
-    foreach ($obj in $j) { 
-
-        # Need to be using at least Visual Studio 15.2 in order to have the appropriate
-        # set of SDK fixes. Parsing the installationName is the only place where this is 
-        # recorded in that form.
-        $name = $obj.installationName
-        if ($name -match "VisualStudio(Preview)?/(.*)") { 
-            if (Test-SupportedVisualStudioVersion $matches[2]) {
-                Write-Output $obj.installationPath
-                Write-Output $obj.instanceId
-                return
-            }
-        }
-        else {
-            Write-Host "Unrecognized installationName format $name"
-        }
-    }
-
-    throw "Could not find a suitable Visual Studio Version"
-}
-
-# Get the directory of the first Visual Studio which meets our minimal 
-# requirements for the Roslyn repo
-function Get-VisualStudioDir() {
-    $both = Get-VisualStudioDirAndId
-    return $both[0]
-}
-
 # Clear out the NuGet package cache
 function Clear-PackageCache() {
     $dotnet = Ensure-DotnetSdk
@@ -392,8 +237,3 @@ function Restore-Project([string]$projectFileName, [string]$logFilePath = "") {
     Exec-Console (Ensure-DotNetSdk) "restore --verbosity quiet $projectFilePath $logArg"
 }
 
-function Unzip-File([string]$zipFilePath, [string]$outputDir) {
-    Add-Type -AssemblyName System.IO.Compression.FileSystem | Out-Null
-    [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $outputDir)
-}
-

build/scripts/build.ps1

@@ -214,7 +214,7 @@ function Make-BootstrapBuild() {
 
     Run-MSBuild $projectPath "/t:Pack /p:DotNetUseShippingVersions=true /p:InitialDefineConstants=BOOTSTRAP /p:PackageOutputPath=$dir" -logFileName "Bootstrap" -useDotnetBuild:$buildCoreClr
     $packageFile = Get-ChildItem -Path $dir -Filter "$packageName.*.nupkg"    
-    Unzip-File "$dir\$packageFile" $dir
+    Unzip "$dir\$packageFile" $dir
 
     Write-Host "Cleaning Bootstrap compiler artifacts"
     Run-MSBuild $projectPath "/t:Clean" -logFileName "BootstrapClean"
@@ -426,9 +426,16 @@ function Test-XUnit() {
 function Deploy-VsixViaTool() { 
     $vsixDir = Get-PackageDir "RoslynTools.VSIXExpInstaller"
     $vsixExe = Join-Path $vsixDir "tools\VsixExpInstaller.exe"
-    $both = Get-VisualStudioDirAndId
-    $vsDir = $both[0].Trim("\")
-    $vsId = $both[1]
+    
+    $vsInfo = LocateVisualStudio
+    if ($vsInfo -eq $null) {
+        throw "Unable to locate required Visual Studio installation"
+    }
+
+    $vsDir = $vsInfo.installationPath.TrimEnd("\")
+    $vsId = $vsInfo.instanceId
+    $vsMajorVersion = $vsInfo.installationVersion.Split('.')[0]
+
     $hive = "RoslynDev"
     Write-Host "Using VS Instance $vsId at `"$vsDir`""
     $baseArgs = "/rootSuffix:$hive /vsInstallDir:`"$vsDir`""
@@ -437,7 +444,7 @@ function Deploy-VsixViaTool() {
 
     # Actual uninstall is failing at the moment using the uninstall options. Temporarily using
     # wildfire to uninstall our VSIX extensions
-    $extDir = Join-Path ${env:USERPROFILE} "AppData\Local\Microsoft\VisualStudio\15.0_$($vsid)$($hive)"
+    $extDir = Join-Path ${env:USERPROFILE} "AppData\Local\Microsoft\VisualStudio\$vsMajorVersion.0_$vsid$hive"
     if (Test-Path $extDir) {
         foreach ($dir in Get-ChildItem -Directory $extDir) {
             $name = Split-Path -leaf $dir
@@ -593,8 +600,7 @@ try {
     }
 
     if ($launch) {
-        $devenvExe = Get-VisualStudioDir
-        $devenvExe = Join-Path $devenvExe 'Common7\IDE\devenv.exe'
+        $devenvExe = Join-Path $env:VSINSTALLDIR 'Common7\IDE\devenv.exe'
         &$devenvExe /rootSuffix RoslynDev
     }
 

build/scripts/locate-vs.ps1

-
-Set-StrictMode -version 2.0
-$ErrorActionPreference="Stop"
-
-try {
-    . (Join-Path $PSScriptRoot "build-utils.ps1")
-    $p = Get-VisualStudioDir
-    Write-Host (Join-Path $p "Common7\Tools\")
-}
-catch {
-  Write-Error $_.Exception.Message
-  # Return an empty string and let the caller fallback or handle this as appropriate
-  return ""
-}

build/scripts/test-machine-setup.ps1

-# Script for testing out the various functions on a given machine.  Useful for
-# debugging Jeknins issues.
-
-param ([switch]$simple = $false)
-
-Set-StrictMode -version 2.0
-$ErrorActionPreference="Stop"
-
-try {
-    if (-not $simple) { 
-        Set-PSDebug -Trace 2
-    }
-
-    . (Join-Path $PSScriptRoot "build-utils.ps1")
-    Write-Host "Calling Get-MSBuildKindAndDir"
-    Get-MSBuildKindAndDir
-    Write-Host "Calling Get-MSBuildKindAndDir -xcopy"
-    Get-MSBuildKindAndDir -xcopy
-    Write-Host "Calling Get-MSBuildDir"
-    Get-MSBuildDir
-    Write-Host "Calling Get-MSBuildDir -xcopy"
-    Get-MSBuildDir -xcopy
-    
-    try {
-        Write-Host "Calling Get-VisualStudioDir"
-        Get-VisualStudioDir
-    }
-    catch { 
-        Write-Host "Unable to find Visual Studio (expected on a machine without VS)"
-        Write-Host $_
-        Write-Host $_.Exception
-    }
-
-    try { 
-
-        $vswhere = Ensure-BasicTool "vswhere" "1.0.50"
-        Write-Host "VSWhere is $vswhere"
-    }
-    catch { 
-        Write-Host "Error gettin vswhere"
-        Write-Host $_
-        Write-Host $_.Exception
-    }
-
-    Write-Host "Enumerate packages dir"
-    $d = Get-PackagesDir
-    Get-ChildItem $d
-
-}
-catch {
-    Write-Host $_.Exception.Message
-    exit 1
-}
-finally {
-    Set-PSDebug -Trace 0
-}

build/scripts/tools.ps1

@@ -7,6 +7,17 @@ set-strictmode -version 2.0
 $ErrorActionPreference = "Stop"
 [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
 
+function Create-Directory([string[]] $path) {
+  if (!(Test-Path $path)) {
+    New-Item -path $path -force -itemType "Directory" | Out-Null
+  }
+}
+
+function Unzip([string]$zipfile, [string]$outpath) {
+  Add-Type -AssemblyName System.IO.Compression.FileSystem
+  [System.IO.Compression.ZipFile]::ExtractToDirectory($zipfile, $outpath)
+}
+
 function InitializeDotNetCli([bool]$install) {
   # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism
   $env:DOTNET_MULTILEVEL_LOOKUP=0
@@ -66,6 +77,143 @@ function InstallDotNetSdk([string] $dotnetRoot, [string] $version) {
   }
 }
 
+#
+# Locates Visual Studio MSBuild installation. 
+# The preference order for MSBuild to use is as follows:
+#
+#   1. MSBuild from an active VS command prompt
+#   2. MSBuild from a compatible VS installation
+#   3. MSBuild from the xcopy tool package
+#
+# Returns full path to msbuild.exe.
+# Throws on failure.
+#
+function InitializeVisualStudioMSBuild {
+  $vsMinVersionStr = if (!$GlobalJson.tools.vs.version) { $GlobalJson.tools.vs.version } else { "15.9" }
+  $vsMinVersion = [Version]::new($vsMinVersionStr) 
+
+  # Try msbuild command available in the environment.
+  if ($env:VSINSTALLDIR -ne $null) {
+    $msbuildCmd = Get-Command "msbuild.exe" -ErrorAction SilentlyContinue
+    if ($msbuildCmd -ne $null) {
+      if ($msbuildCmd.Version -ge $vsMinVersion) {
+        return $msbuildCmd.Path
+      }
+
+      # Report error - the developer environment is initialized with incompatible VS version.
+      throw "Developer Command Prompt for VS $($env:VisualStudioVersion) is not recent enough. Please upgrade to $vsMinVersionStr or build from a plain CMD window"
+    }
+  }
+
+  # Locate Visual Studio installation or download x-copy msbuild.
+  $vsInfo = LocateVisualStudio  
+  if ($vsInfo -ne $null) {
+    $vsInstallDir = $vsInfo.installationPath
+    $vsMajorVersion = $vsInfo.installationVersion.Split('.')[0]
+
+    InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion
+  } else {
+
+    if (Get-Member -InputObject $GlobalJson.tools -Name "xcopy-msbuild") {
+      $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild'
+      $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0]
+    } else {
+      $vsMajorVersion = $vsMinVersion.Major
+      $xcopyMSBuildVersion = "$vsMajorVersion.$($vsMinVersion.Minor).0-alpha"
+    }
+
+    $vsInstallDir = InstallXCopyMSBuild $xcopyMSBuildVersion
+  }
+
+  $msbuildVersionDir = if ([int]$vsMajorVersion -lt 16) { "$vsMajorVersion.0" } else { "Current" }
+  return Join-Path $vsInstallDir "MSBuild\$msbuildVersionDir\Bin\msbuild.exe"
+}
+
+function InitializeVisualStudioEnvironmentVariables([string] $vsInstallDir, [string] $vsMajorVersion) {
+  $env:VSINSTALLDIR = $vsInstallDir
+  Set-Item "env:VS$($vsMajorVersion)0COMNTOOLS" (Join-Path $vsInstallDir "Common7\Tools\")
+  
+  $vsSdkInstallDir = Join-Path $vsInstallDir "VSSDK\"
+  if (Test-Path $vsSdkInstallDir) {
+    Set-Item "env:VSSDK$($vsMajorVersion)0Install" $vsSdkInstallDir
+    $env:VSSDKInstall = $vsSdkInstallDir
+  }
+}
+
+function InstallXCopyMSBuild([string] $packageVersion) {
+  $toolsRoot = GetToolsRoot
+  $packageName = "RoslynTools.MSBuild"
+  $packageDir = Join-Path $toolsRoot "msbuild\$packageVersion"
+  $packagePath = Join-Path $packageDir "$packageName.$packageVersion.nupkg"
+
+  if (!(Test-Path $packageDir)) {
+    Create-Directory $packageDir
+    Write-Host "Downloading $packageName $packageVersion"
+    Invoke-WebRequest "https://dotnet.myget.org/F/roslyn-tools/api/v2/package/$packageName/$packageVersion/" -OutFile $packagePath
+    Unzip $packagePath $packageDir
+  }
+
+  return Join-Path $packageDir "tools"
+}
+
+#
+# Locates Visual Studio instance that meets the minimal requirements specified by tools.vs object in global.json.
+#
+# The following properties of tools.vs are recognized:
+#   "version": "{major}.{minor}"    
+#       Two part minimal VS version, e.g. "15.9", "16.0", etc.
+#   "components": ["componentId1", "componentId2", ...] 
+#       Array of ids of workload components that must be available in the VS instance.
+#       See e.g. https://docs.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-enterprise?view=vs-2017
+#
+# Returns JSON describing the located VS instance (same format as returned by vswhere), 
+# or $null if no instance meeting the requirements is found on the machine.
+#
+function LocateVisualStudio {
+  $vswhereVersion = Get-Member -InputObject $GlobalJson.tools -Name "vswhere"
+  if ($vsWhereVersion -eq $null) {
+    $vswhereVersion = "2.5.2"
+  }
+
+  $toolsRoot = GetToolsRoot
+  $vsWhereDir = Join-Path $toolsRoot "vswhere\$vswhereVersion"
+  $vsWhereExe = Join-Path $vsWhereDir "vswhere.exe"
+
+  if (!(Test-Path $vsWhereExe)) {
+    Create-Directory $vsWhereDir
+    Write-Host "Downloading vswhere"
+    Invoke-WebRequest "https://github.com/Microsoft/vswhere/releases/download/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe
+  }
+
+  $vs = $GlobalJson.tools.vs
+  $args = @("-latest", "-prerelease", "-format", "json", "-requires", "Microsoft.Component.MSBuild")
+  
+  if (Get-Member -InputObject $vs -Name "version") { 
+    $args += "-version"
+    $args += $vs.version
+  }
+
+  if (Get-Member -InputObject $vs -Name "components") { 
+    foreach ($component in $vs.components) {
+      $args += "-requires"
+      $args += $component
+    }    
+  }
+
+  $vsInfo =& $vsWhereExe $args | ConvertFrom-Json
+
+  if ($lastExitCode -ne 0) {
+    return $null
+  }
+
+  # use first matching instance
+  return $vsInfo[0]
+}
+
+function GetToolsRoot() {
+  return Join-Path $RepoRoot ".tools"
+}
+
 function ExitWithExitCode([int] $exitCode) {
   exit $exitCode
 }

global.json

 {
   "tools": {
-    "dotnet": "2.1.401"
+    "dotnet": "2.1.401",
+    "vs": {
+      "version": "15.8"
+    },
+    "xcopy-msbuild": "15.9.0-alpha"
   }
 }