Consume centrally trained optprof data (#40510)

* Rework build definitions to accept VS optprof drop

azure-pipelines-official.yml

@@ -8,8 +8,7 @@ resources:
 #  SignType: real
 #  SkipTests: false
 #  SkipApplyOptimizationData: false
-#  IbcSourceBranchName: 'default'
-#  IbcDropId: 'default'
+#  IbcDrop: 'default'
 
 # The variables `_DotNetArtifactsCategory` and `_DotNetValidationArtifactsCategory` are required for proper publishing of build artifacts. See https://github.com/dotnet/roslyn/pull/38259
 variables:
@@ -76,8 +75,7 @@ stages:
                 -officialSkipTests $(SkipTests)
                 -officialSkipApplyOptimizationData $(SkipApplyOptimizationData)
                 -officialSourceBranchName $(SourceBranchName)
-                -officialIbcSourceBranchName $(IbcSourceBranchName)
-                -officialIbcDropId $(IbcDropId)
+                -officialIbcDrop $(IbcDrop)
                 -skipAnalyzers
                 /p:RepositoryName=$(Build.Repository.Name)
                 /p:VisualStudioDropAccessToken=$(System.AccessToken)

eng/InternalTools.props

+<!-- Copyright (c)  Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information. -->
+<Project>
+   
+    <PropertyGroup>
+        <RestoreSources Condition="'$(UsingToolVisualStudioIbcTraining)' == 'true'">
+          $(RestoreSources);
+          https://devdiv.pkgs.visualstudio.com/_packaging/Engineering/nuget/v3/index.json;
+          https://api.nuget.org/v3/index.json;
+        </RestoreSources>
+    </PropertyGroup>
+    
+    <ItemGroup>
+        <PackageReference Include="Microsoft.DevDiv.Optimization.Data.PowerShell" Version="$(MicrosoftDevDivOptimizationDataPowerShellVersion)" ExcludeAssets="all" Condition="'$(UsingToolVisualStudioIbcTraining)' == 'true'"/>
+    </ItemGroup>
+
+</Project>
\ No newline at end of file

eng/Versions.props

@@ -66,6 +66,7 @@
     <MicrosoftCodeQualityAnalyzersVersion>$(RoslynDiagnosticsNugetPackageVersion)</MicrosoftCodeQualityAnalyzersVersion>
     <SystemCompositionVersion>1.0.31</SystemCompositionVersion>
     <MicrosoftCSharpVersion>4.3.0</MicrosoftCSharpVersion>
+    <MicrosoftDevDivOptimizationDataPowerShellVersion>1.0.210</MicrosoftDevDivOptimizationDataPowerShellVersion>
     <MicrosoftDiagnosticsRuntimeVersion>0.8.31-beta</MicrosoftDiagnosticsRuntimeVersion>
     <MicrosoftDiagnosticsTracingTraceEventVersion>1.0.35</MicrosoftDiagnosticsTracingTraceEventVersion>
     <MicrosoftDiaSymReaderVersion>1.3.0</MicrosoftDiaSymReaderVersion>

eng/build.ps1

@@ -49,8 +49,7 @@ param (
   [string]$officialSkipApplyOptimizationData = "",
   [string]$officialSkipTests = "",
   [string]$officialSourceBranchName = "",
-  [string]$officialIbcSourceBranchName = "",
-  [string]$officialIbcDropId = "",
+  [string]$officialIbcDrop = "",
 
   # Test actions
   [switch]$test32,
@@ -108,10 +107,8 @@ function Print-Usage() {
   Write-Host "  -officialSkipTests <bool>                   Pass 'true' to not run tests"
   Write-Host "  -officialSkipApplyOptimizationData <bool>   Pass 'true' to not apply optimization data"
   Write-Host "  -officialSourceBranchName <string>          The source branch name"
-  Write-Host "  -officialIbcDropId <string>                 IBC data drop to use (e.g. '20190210.1/935479/1')."
+  Write-Host "  -officialIbcDrop <string>                   IBC data drop to use (e.g. 'ProfilingOutputs/DevDiv/VS/..')."
   Write-Host "                                              'default' for the most recent available for the branch."
-  Write-Host "  -officialIbcSourceBranchName <string>       IBC source branch (e.g. 'master-vs-deps')"
-  Write-Host "                                              'default' to select branch based on eng/config/PublishData.json."
   Write-Host ""
   Write-Host "Command line arguments starting with '/p:' are passed through to MSBuild."
 }
@@ -146,8 +143,6 @@ function Process-Arguments() {
   OfficialBuildOnly "officialSkipTests"
   OfficialBuildOnly "officialSkipApplyOptimizationData"
   OfficialBuildOnly "officialSourceBranchName"
-  OfficialBuildOnly "officialIbcDropId"
-  OfficialBuildOnly "officialIbcSourceBranchName"
 
   if ($officialBuildId) {
     $script:useGlobalNuGetCache = $false
@@ -220,8 +215,7 @@ function BuildSolution() {
 
   $testTargetFrameworks = if ($testCoreClr) { "netcoreapp3.0%3Bnetcoreapp2.1" } else { "" }
   
-  $ibcSourceBranchName = GetIbcSourceBranchName
-  $ibcDropId = if ($officialIbcDropId -ne "default") { $officialIbcDropId } else { "" }
+  $ibcDropName = GetIbcDropName
 
   # Do not set this property to true explicitly, since that would override values set in projects.
   $suppressExtensionDeployment = if (!$deployExtensions) { "/p:DeployExtension=false" } else { "" } 
@@ -257,10 +251,9 @@ function BuildSolution() {
       /p:BootstrapBuildPath=$bootstrapDir `
       /p:TestTargetFrameworks=$testTargetFrameworks `
       /p:TreatWarningsAsErrors=true `
-      /p:VisualStudioIbcSourceBranchName=$ibcSourceBranchName `
-      /p:VisualStudioIbcDropId=$ibcDropId `
       /p:EnableNgenOptimization=$applyOptimizationData `
       /p:IbcOptimizationDataDir=$ibcDir `
+      /p:VisualStudioIbcDrop=$ibcDropName `
       $suppressExtensionDeployment `
       $msbuildWarnAsError `
       $buildFromSource `
@@ -280,33 +273,42 @@ function GetIbcSourceBranchName() {
   }
 
   function calculate {
-    $fallback = "master-vs-deps"
-
-    if (!$officialIbcSourceBranchName) {
-      return $fallback
-    }  
-
-    if ($officialIbcSourceBranchName -ne "default") {
-      return $officialIbcSourceBranchName
-    }
+    $fallback = "master"
 
     $branchData = GetBranchPublishData $officialSourceBranchName
     if ($branchData -eq $null) {
       Write-Host "Warning: Branch $officialSourceBranchName is not listed in PublishData.json. Using IBC data from '$fallback'." -ForegroundColor Yellow
-      Write-Host "Override by setting IbcSourceBranchName build variable." -ForegroundColor Yellow
+      Write-Host "Override by setting IbcDrop build variable." -ForegroundColor Yellow
       return $fallback
     }
 
-    if (Get-Member -InputObject $branchData -Name "ibcSourceBranch") {
-      return $branchData.ibcSourceBranch 
-    }
-
-    return $officialSourceBranchName
+    return $branchData.vsBranch
   }
 
   return $global:_IbcSourceBranchName = calculate
 }
 
+function GetIbcDropName() {
+
+    if ($officialIbcDrop -and $officialIbcDrop -ne "default"){
+        return $officialIbcDrop
+    }
+
+    # Don't try and get the ibc drop if we're not in an official build as it won't be used anyway
+    if (!$officialBuildId) {
+        return ""
+    }
+
+    # Bring in the ibc tools
+    $packagePath = Join-Path (Get-PackageDir "Microsoft.DevDiv.Optimization.Data.PowerShell") "lib\net461"
+    Import-Module (Join-Path $packagePath "Optimization.Data.PowerShell.dll")
+    
+    # Find the matching drop
+    $branch = GetIbcSourceBranchName
+    $drop = Find-OptimizationInputsStoreForBranch -ProjectName "DevDiv" -RepositoryName "VS" -BranchName $branch
+    return $drop.Name
+}
+
 # Set VSO variables used by MicroBuildBuildVSBootstrapper pipeline task
 function SetVisualStudioBootstrapperBuildArgs() {
   $fallbackBranch = "master-vs-deps"

eng/config/PublishData.json

@@ -79,8 +79,7 @@
       "vsix": [ "https://dotnet.myget.org/F/roslyn-nonnull/vsix/upload" ],
       "channels": [ "nonnull" ],
       "vsBranch": "lab/d16.1stg",
-      "vsMajorVersion": 16,
-      "ibcSourceBranch": "master-vs-deps"
+      "vsMajorVersion": 16
     },
     "features/dataflow": {
       "nugetKind": "PerBuildPreRelease",
@@ -89,8 +88,7 @@
       "vsix": [ "https://dotnet.myget.org/F/roslyn/vsix/upload" ],
       "channels": [ "dataflow" ],
       "vsBranch": "lab/d16.1stg",
-      "vsMajorVersion": 16,
-      "ibcSourceBranch": "master-vs-deps"
+      "vsMajorVersion": 16
     },
     "features/razorSupport2": {
       "nugetKind": [ "Shipping", "NonShipping" ],
@@ -99,8 +97,7 @@
       "vsix": [ "https://dotnet.myget.org/F/roslyn/vsix/upload" ],
       "channels": [ "razorSupport2" ],
       "vsBranch": "lab/d16.1stg",
-      "vsMajorVersion": 16,
-      "ibcSourceBranch": "master-vs-deps"
+      "vsMajorVersion": 16
     },
     "features/compilerNext": {
       "nugetKind": [ "Shipping", "NonShipping" ],
@@ -109,8 +106,7 @@
       "vsix": [ "https://dotnet.myget.org/F/roslyn/vsix/upload" ],
       "channels": [ "compilerNext" ],
       "vsBranch": "master",
-      "vsMajorVersion": 16,
-      "ibcSourceBranch": "master-vs-deps"
+      "vsMajorVersion": 16
     },
     "features/dotnetFormat": {
       "nugetKind": [ "Shipping", "NonShipping" ],
@@ -119,8 +115,7 @@
       "vsix": [ "https://dotnet.myget.org/F/roslyn/vsix/upload" ],
       "channels": [ "dotnetFormat" ],
       "vsBranch": "master",
-      "vsMajorVersion": 16,
-      "ibcSourceBranch": "master-vs-deps"
+      "vsMajorVersion": 16
     }
   },
   "releases": {