Apply optimization data to managed dependencies inserted to VS

07a65bde · Tomas Matousek · 3d44e4f7 · 07a65bde · 07a65bde · 07a65bde
7 changed file
--- a/build/Targets/Packages.props
+++ b/build/Targets/Packages.props
@@ -63,7 +63,7 @@
    <MicrosoftDiaSymReaderConverterXmlVersion>1.1.0-beta1-63314-01</MicrosoftDiaSymReaderConverterXmlVersion>
    <MicrosoftDiaSymReaderNativeVersion>1.7.0</MicrosoftDiaSymReaderNativeVersion>
    <MicrosoftDiaSymReaderPortablePdbVersion>1.5.0</MicrosoftDiaSymReaderPortablePdbVersion>
-    <MicrosoftDotNetIBCMerge>4.7.2-alpha-00001</MicrosoftDotNetIBCMerge>
+    <MicrosoftDotNetIBCMergeVersion>4.7.2-alpha-00001</MicrosoftDotNetIBCMergeVersion>
    <MicrosoftDotNetVersionToolsVersion>1.0.27-prerelease-01811-02</MicrosoftDotNetVersionToolsVersion>
    <MicrosoftDotNetBuildTasksFeedVersion>2.1.0-prerelease-02419-02</MicrosoftDotNetBuildTasksFeedVersion>
    <MicrosoftDotNetSignToolVersion>1.0.0-beta.18519.9</MicrosoftDotNetSignToolVersion>
@@ -165,7 +165,7 @@
    <MicrosoftDiaSymReaderPdb2PdbVersion>1.1.0-beta1-62506-02</MicrosoftDiaSymReaderPdb2PdbVersion>
    <RestSharpVersion>105.2.3</RestSharpVersion>
    <RoslynBuildUtilVersion>0.9.8-beta</RoslynBuildUtilVersion>
-    <RoslynDependenciesOptimizationDataVersion>2.11.0-beta1-63312-06</RoslynDependenciesOptimizationDataVersion>
+    <RoslynDependenciesOptimizationDataVersion>2.11.0-beta3.20181025.2</RoslynDependenciesOptimizationDataVersion>
    <RoslynDiagnosticsAnalyzersVersion>$(RoslynDiagnosticsNugetPackageVersion)</RoslynDiagnosticsAnalyzersVersion>
    <RoslynToolsMicrosoftLocateVSVersion>0.2.4-beta</RoslynToolsMicrosoftLocateVSVersion>
    <RoslynToolsVSIXExpInstallerVersion>1.0.0-beta2-63222-01</RoslynToolsVSIXExpInstallerVersion>

--- a/build/Targets/RepoToolset/OptimizationData.targets
+++ b/build/Targets/RepoToolset/OptimizationData.targets
 <?xml version="1.0" encoding="utf-8"?>
-<!-- Copyright (c)  Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information. -->
+<!-- All Rights Reserved. Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. See the LICENSE file in the project root for more information. -->
 <Project>

  <PropertyGroup>
    <MSBuildAllProjects>$(MSBuildAllProjects);$(MSBuildThisFileFullPath)</MSBuildAllProjects>
  </PropertyGroup>
-  
-  <!-- 
-    Support for in-place modification of the compiled binary.
-    Since there may be more than one post compile modification, we must take care not
-    to break incremental builds. A timestemp file is written out when all modification
-    targets have completed.
-    Note that the targets participating in post compile modification must list the
-    sentinel file as one of their outputs, but they should not modify it.
-   -->

+  <!--
+    Required properties:
+      IbcOptimizationDataDir           The directory containing IBC optimization data.
+  -->
  <PropertyGroup>
+    <_PreviousOptimizedFile>$([System.IO.Path]::Combine($(IbcOptimizationDataDir), '$(TargetName).pgo'))</_PreviousOptimizedFile>
    <PostCompileBinaryModificationSentinelFile>$(IntermediateOutputPath)$(TargetFileName).pcbm</PostCompileBinaryModificationSentinelFile>
-    <OptimizationDataFolderPath>$(NuGetPackageRoot)\RoslynDependencies.OptimizationData\$(RoslynDependenciesOptimizationDataVersion)\content\OptimizationData</OptimizationDataFolderPath>
-    <OptimizationDataFile>$([System.IO.Path]::GetFullPath('$(OptimizationDataFolderPath)\$(TargetName).pgo'))</OptimizationDataFile>
-    <IbcMergePath>$(NuGetPackageRoot)\Microsoft.DotNet.IBCMerge\$(MicrosoftDotNetIBCMerge)\lib\net45\ibcmerge.exe</IbcMergePath>
  </PropertyGroup>
-  
+
+  <!--
+    We need to write out this sentinel file so that when ApplyOptimizations runs and compares the intermediate assemby location
+    against itself the PostCompileBinaryModificationSentinelFile will have a newer timestamp allowing the target to be run.
+  -->
  <Target Name="PostCompileBinaryModification"
          AfterTargets="CoreCompile"
-          DependsOnTargets="ApplyOptimizations"
-          Inputs="@(IntermediateAssembly)"
+          DependsOnTargets="_InitializeAssemblyOptimizationWithTargetAssembly;ApplyOptimizations"
+          Condition="'$(IsWpfTempProject)' != 'true' and Exists('$(_PreviousOptimizedFile)')"
+          Inputs="$(MSBuildAllProjects);@(IntermediateAssembly)"
          Outputs="@(IntermediateAssembly);$(PostCompileBinaryModificationSentinelFile)">

    <!-- Write out a sentinel timestamp file to prevent unnecessary work in incremental builds. -->
@@ -36,27 +34,52 @@
    </ItemGroup>
  </Target>

-  <!-- The IBCMerge tool is internal only and hence only available during official Microbuild runs.  This target is 
-       specifically not gated on the existance of this file.  During an official build it shoudl always be present and 
-       if it's not then an error needs to be raised.
+  <Target Name="_InitializeAssemblyOptimizationWithTargetAssembly">
+    <ItemGroup>
+      <OptimizeAssembly Include="@(IntermediateAssembly)" PreviousOptimizedFile="$(_PreviousOptimizedFile)" />
+    </ItemGroup>
+  </Target>
+
+  <!--
+    Merges optimization data to assemblies specified in OptimizeAssembly item group.
+
+    Non-incremental. Calling targets need to handle incremental build if necessary.
+    Runs during any CI build. Performs the actual merge only when IBCMerge tool is available. It is expected to be available in an official build.
+  -->
+  <Target Name="_CalculateIbcArgs">
+    <ItemGroup>
+      <OptimizeAssembly>
+        <!--
+          -delete to delete data previously embedded in the binary. This is a no-op for binaries produced by this build, but is needed for dependencies such as System.Reflection.Metadata.
+          -incremental to map data stored in the optimized binary, which comes from a previous build, to the new build of the binary.
+        -->
+        <_IbcArgs>-q -f -partialNGEN -minify -delete -mo "%(OptimizeAssembly.Identity)" -incremental "%(OptimizeAssembly.PreviousOptimizedFile)"</_IbcArgs>
+      </OptimizeAssembly>
+    </ItemGroup>
+  </Target>

-       A local build emulating an official build can pass /p:SkipApplyOptimizations=true to avoid this error.
-       -->
  <Target Name="ApplyOptimizations"
-          Condition="'$(OfficialBuild)' == 'true' AND '$(IsShipping)' == 'true' AND '$(SkipApplyOptimizations)' != 'true' AND Exists('$(OptimizationDataFile)')"
-          Inputs="@(IntermediateAssembly)"
-          Outputs="@(IntermediateAssembly);$(PostCompileBinaryModificationSentinelFile)">
-    <Message Text="Adding optimization data to @(IntermediateAssembly)" />
+          DependsOnTargets="_CalculateIbcArgs"
+          Condition="'@(OptimizeAssembly)' != '' and '$(Configuration)' == 'Release' and '$(ContinuousIntegrationBuild)' == 'true'">
+    
+    <PropertyGroup>
+      <_IbcMergePath>$(NuGetPackageRoot)microsoft.dotnet.ibcmerge\$(MicrosoftDotNetIBCMergeVersion)\lib\net45\ibcmerge.exe</_IbcMergePath>
+
+      <_RunIbcMerge>false</_RunIbcMerge>
+      <_RunIbcMerge Condition="'$(OfficialBuild)' == 'true' or Exists('$(_IbcMergePath)')">true</_RunIbcMerge>
+    </PropertyGroup>

-    <Error Text="IBCMerge not found at $(IbcMergePath). Local developer builds should pass /p:SkipApplyOptimizations=true to avoid this"
-           Condition="!Exists('$(IbcMergePath)')" />
+    <Message Text='IBCMerge tool will be run in an official build with arguments: %(OptimizeAssembly._IbcArgs)'
+             Condition="'$(_RunIbcMerge)' != 'true'" 
+             Importance="normal"/>

-    <Exec Command="&quot;$(IbcMergePath)&quot; -q -f -partialNGEN -minify -mo &quot;@(IntermediateAssembly)&quot; -excludeGenmethUnboxingStubs -incremental &quot;$(OptimizationDataFile)&quot;"
+    <Exec Command='"$(_IbcMergePath)" %(OptimizeAssembly._IbcArgs)' 
          ConsoleToMSBuild="true"
-          Condition="Exists('$(IbcMergePath)')">
+          Condition="'$(_RunIbcMerge)' == 'true'">
      <Output TaskParameter="ConsoleOutput" PropertyName="IbcMergeOutput" />
    </Exec>

    <Message Text="$(IbcMergeOutput)" />
  </Target>
+
 </Project>
\ No newline at end of file
--- a/build/Targets/Settings.props
+++ b/build/Targets/Settings.props
@@ -46,6 +46,8 @@
    <VisualStudioReferenceAssemblyVersion Condition="'$(VisualStudioReferenceAssemblyVersion)' == ''">$(VisualStudioReferenceMajorVersion).0.0.0</VisualStudioReferenceAssemblyVersion>
    <MinimumVisualStudioVersion>$(VisualStudioVersion)</MinimumVisualStudioVersion>

+    <IbcOptimizationDataDir>$(NuGetPackageRoot)roslyndependencies.optimizationdata\$(RoslynDependenciesOptimizationDataVersion)\content\OptimizationData\</IbcOptimizationDataDir>
+
    <!-- Disable AppX packaging for the Roslyn source. Not setting this to false has the side effect
         that any builds of portable projects end up in a sub folder of $(OutputPath). Search for this flag in
         Microsoft.Common.CurrentVersion.targets to see how it is consumed -->

--- a/build/Targets/Signing.props
+++ b/build/Targets/Signing.props
@@ -36,11 +36,13 @@
  </ItemGroup>

  <!--
-     Add CoreXT packages to the set of packages to sign.
-     The toolset already includes NuGet and VSIX packages built to standard output directories.
+    Add CoreXT packages to the set of packages to sign.
+    The toolset already includes NuGet and VSIX packages built to standard output directories.
+    We also need to re-sign packages in ManagedDependencies that we updated with optimization data.
  -->
  <ItemGroup>
    <ItemsToSign Include="$(ArtifactsConfigurationDir)DevDivPackages\Roslyn\*.nupkg" />
+    <ItemsToSign Include="$(ArtifactsConfigurationDir)DevDivPackages\ManagedDependencies\*-vs-*.nupkg" />
  </ItemGroup>

 </Project>
--- a/build/ToolsetPackages/InternalToolset.csproj
+++ b/build/ToolsetPackages/InternalToolset.csproj
@@ -5,6 +5,6 @@
    <TargetFramework>net472</TargetFramework>
  </PropertyGroup>
  <ItemGroup>
-    <PackageReference Include="Microsoft.DotNet.IBCMerge" Version="$(MicrosoftDotNetIBCMerge)" />
+    <PackageReference Include="Microsoft.DotNet.IBCMerge" Version="$(MicrosoftDotNetIBCMergeVersion)" />
  </ItemGroup>
 </Project>
--- a/build/scripts/build.ps1
+++ b/build/scripts/build.ps1
@@ -255,9 +255,29 @@ function Build-Artifacts() {

    if ($build -and $pack -and (-not $buildCoreClr)) {
        Build-Installer
+        Build-OptProfData
    }
 }

+function Build-OptProfData() {
+    $optProfToolDir = Get-PackageDir "RoslynTools.OptProf"
+    $optProfToolExe = Join-Path $optProfToolDir "tools\roslyn.optprof.exe"
+    $configFile = Join-Path $repoDir "build\config\optprof.json"
+    $insertionFolder = Join-Path $vsSetupDir "Insertion"
+    $outputFolder = Join-Path $configDir "DevDivInsertionFiles\OptProf"
+    Write-Host "Generating optprof data using '$configFile' into '$outputFolder'"
+    $optProfArgs = "--configFile $configFile --insertionFolder $insertionFolder --outputFolder $outputFolder"
+    Exec-Console $optProfToolExe $optProfArgs
+
+    # Write Out Branch we are inserting into
+    $vsBranchFolder = Join-Path $configDir "DevDivInsertionFiles\BranchInfo"
+    New-Item -ItemType Directory -Force -Path $vsBranchFolder
+    $vsBranchText = Join-Path $vsBranchFolder "vsbranch.txt"
+    # InsertTargetBranchFullName is defined in .vsts-ci.yml
+    $vsBranch = $Env:InsertTargetBranchFullName
+    $vsBranch >> $vsBranchText
+}
+
 function Build-Installer () {
    ## Copying Artifacts
    $installerDir = Join-Path $configDir "Installer"

--- a/src/Setup/DevDivInsertionFiles/DevDivInsertionFiles.csproj
+++ b/src/Setup/DevDivInsertionFiles/DevDivInsertionFiles.csproj
@@ -29,21 +29,26 @@
    <_DevDivInsertionFilesDir>$(ArtifactsConfigurationDir)DevDivInsertionFiles\</_DevDivInsertionFilesDir>
    <_DependentAssemblyVersionsFile>$(_DevDivInsertionFilesDir)DependentAssemblyVersions.csv</_DependentAssemblyVersionsFile>
    <_ManagedDependenciesDir>$(ArtifactsConfigurationDir)DevDivPackages\ManagedDependencies\</_ManagedDependenciesDir>
+    <_OptimizedDependenciesDir>$(ArtifactsTmpDir)OptimizedDependencies\</_OptimizedDependenciesDir>
+    <_OptimizedNuGetPackageVersionSuffix Condition="'$(OfficialBuild)' != 'true'">vs-ci</_OptimizedNuGetPackageVersionSuffix>
+    <_OptimizedNuGetPackageVersionSuffix Condition="'$(OfficialBuild)' == 'true'">vs-$(VersionSuffixDateStamp)-$(VersionSuffixBuildOfTheDayPadded)</_OptimizedNuGetPackageVersionSuffix>
  </PropertyGroup>

  <!-- 
    List of dependencies that are expected to be inserted into VS by Roslyn Insertion Tool.
+    DevEnvLib is the relative path within the package of the binary that's used by devenv process.
+    This binary will get IBC data embedded if they are available for it.
  -->
  <ItemGroup>
-    <ExpectedDependency Include="Humanizer.Core"/>
-    <ExpectedDependency Include="ICSharpCode.Decompiler"/>
-    <ExpectedDependency Include="Microsoft.DiaSymReader"/>
-    <ExpectedDependency Include="Microsoft.CodeAnalysis.Elfie"/>
-    <ExpectedDependency Include="System.Collections.Immutable"/>
-    <ExpectedDependency Include="System.Reflection.Metadata"/>
-    <ExpectedDependency Include="System.Runtime.CompilerServices.Unsafe"/>
-    <ExpectedDependency Include="System.Text.Encoding.CodePages"/>
-    <ExpectedDependency Include="System.Threading.Tasks.Extensions"/>
+    <ExpectedDependency Include="Humanizer.Core" DevEnvLib="lib/netstandard2.0/Humanizer.dll"/>
+    <ExpectedDependency Include="ICSharpCode.Decompiler" DevEnvLib="lib/netstandard2.0/ICSharpCode.Decompiler.dll"/>
+    <ExpectedDependency Include="Microsoft.DiaSymReader" DevEnvLib="lib/netstandard1.1/Microsoft.DiaSymReader.dll"/>
+    <ExpectedDependency Include="Microsoft.CodeAnalysis.Elfie" DevEnvLib="lib/net46/Microsoft.CodeAnalysis.Elfie.dll"/>
+    <ExpectedDependency Include="System.Collections.Immutable" DevEnvLib="lib/netstandard2.0/System.Collections.Immutable.dll"/>
+    <ExpectedDependency Include="System.Reflection.Metadata" DevEnvLib="lib/netstandard2.0/System.Reflection.Metadata.dll"/>
+    <ExpectedDependency Include="System.Runtime.CompilerServices.Unsafe" DevEnvLib="lib/netstandard2.0/System.Runtime.CompilerServices.Unsafe.dll"/>
+    <ExpectedDependency Include="System.Text.Encoding.CodePages" DevEnvLib="lib/netstandard2.0/System.Text.Encoding.CodePages.dll"/>
+    <ExpectedDependency Include="System.Threading.Tasks.Extensions" DevEnvLib="lib/netstandard2.0/System.Threading.Tasks.Extensions.dll"/>
  </ItemGroup>
  
  <Target Name="_CalculateDependenciesToInsert" DependsOnTargets="ResolveAssemblyReferences">
@@ -87,15 +92,17 @@

    <!-- Calculate metadata we need -->
    <ItemGroup>
-      <_Dependency Update="@(_Dependency)">
+      <_Dependency>
        <_AssemblyName>$([MSBuild]::ValueOrDefault('%(_Dependency.FusionName)', '').Split(',')[0])</_AssemblyName>
        <_AssemblyVersion>$([MSBuild]::ValueOrDefault('%(_Dependency.FusionName)', '').Split(',')[1])</_AssemblyVersion>
        <_NuGetPackageFileName>%(_Dependency.NuGetPackageId).%(_Dependency.NuGetPackageVersion).nupkg</_NuGetPackageFileName>
-        <_NuGetPackagePath>$(NuGetPackageRoot)$([MSBuild]::ValueOrDefault('%(_Dependency.NuGetPackageId)', '').ToLower())\%(_Dependency.NuGetPackageVersion)\%(_Dependency.NuGetPackageId).%(_Dependency.NuGetPackageVersion).nupkg</_NuGetPackagePath>
+        <_NuGetPackageDir>$(NuGetPackageRoot)$([MSBuild]::ValueOrDefault('%(_Dependency.NuGetPackageId)', '').ToLower())\%(_Dependency.NuGetPackageVersion)\</_NuGetPackageDir>
      </_Dependency>
    </ItemGroup>

-    <!-- Sanity check that prevents us from accidentally inserting a new dependency without knowing. -->
+    <!--
+      Sanity check that prevents us from accidentally inserting a new dependency without knowing.
+    -->
    <ItemGroup>
      <_RemainingDependency Include="@(_Dependency)" Exclude="@(ExpectedDependency)"/>
      <_UninsertedExpectedDependency Include="@(ExpectedDependency)" Exclude="@(_Dependency)"/>
@@ -103,6 +110,26 @@

    <Error Text="Unexpected dependencies found -- update ExpectedDependency list: @(_RemainingDependency)" Condition="'@(_RemainingDependency)' != ''"/>
    <Error Text="Missing dependencies expected to be inserted -- update ExpectedDependency list: @(_UninsertedExpectedDependency)" Condition="'@(_UninsertedExpectedDependency)' != ''"/>
+
+    <!-- 
+      Determine which dependencies to embed optimization data to and calculate information needed for repacking them. 
+    -->
+    <JoinItems Left="@(_Dependency)" LeftMetadata="*" Right="@(ExpectedDependency)" RightMetadata="*">
+      <Output TaskParameter="JoinResult" ItemName="_DependencyWithExpectedMetadata" />
+    </JoinItems>
+
+    <ItemGroup>
+      <_DependencyWithExpectedMetadata>
+        <_UnpackDir>$(_OptimizedDependenciesDir)%(_DependencyWithExpectedMetadata.Identity)\</_UnpackDir>
+        <_OptimizeAssembly>$(_OptimizedDependenciesDir)%(_DependencyWithExpectedMetadata.Identity)\%(_DependencyWithExpectedMetadata.DevEnvLib)</_OptimizeAssembly>
+        <_PreviousOptimizedFile>$([System.IO.Path]::Combine($(IbcOptimizationDataDir), '$([System.IO.Path]::GetFileNameWithoutExtension(%(_DependencyWithExpectedMetadata.DevEnvLib))).pgo'))</_PreviousOptimizedFile>
+      </_DependencyWithExpectedMetadata>
+
+      <_DependencyWithOptimizationData Include="@(_DependencyWithExpectedMetadata)" Condition="Exists(%(_DependencyWithExpectedMetadata._PreviousOptimizedFile))" />
+      <_DependencyWithoutOptimizationData Include="@(_Dependency)" Exclude="@(_DependencyWithOptimizationData)"/>
+
+      <OptimizeAssembly Include="%(_DependencyWithOptimizationData._OptimizeAssembly)" PreviousOptimizedFile="%(_DependencyWithOptimizationData._PreviousOptimizedFile)" />
+    </ItemGroup>
  </Target>

  <!-- 
@@ -122,8 +149,6 @@
    </ItemGroup>
  </Target>

-<<<<<<< HEAD
-=======
  <!-- 
    Unpack dependent packages to a temp folder where optimization data will be applied.
  -->
@@ -134,22 +159,29 @@
                       DestinationFolder="%(_DependencyWithOptimizationData._UnpackDir)" />
  </Target>

->>>>>>> bbe1bc535b... DDIF Fixes
  <!--
    Copy NuGet packages to be inserted into VS by the insertion tool.
  -->
  <Target Name="_CopyPackagesToInsert"
          AfterTargets="Build"
-          DependsOnTargets="_CalculateDependenciesToInsert"
-          Inputs="$(MSBuildAllProjects)"
-          Outputs="@(_Dependency->'$(_ManagedDependenciesDir)%(_NuGetPackageFileName)')"
-          Condition="'$(OfficialBuild)' == 'true'">
+          Condition="'$(Configuration)' == 'Release' and '$(ContinuousIntegrationBuild)' == 'true'"
+          DependsOnTargets="_CalculateDependenciesToInsert;_UnpackOptimizedDependencies;ApplyOptimizations">

    <MakeDir Directories="$(_ManagedDependenciesDir)"/>
-    <Copy SourceFiles="@(_Dependency->'%(_NuGetPackagePath)')" DestinationFolder="$(_ManagedDependenciesDir)"/>

-    <ItemGroup>
-      <FileWrites Include="@(_Dependency->'$(_ManagedDependenciesDir)%(_NuGetPackageFileName)')"/>
-    </ItemGroup>
+    <!-- Repack optimized dependencies -->
+    <Roslyn.Tools.ReplacePackageParts SourcePackage="%(_DependencyWithOptimizationData._NuGetPackageDir)%(_DependencyWithOptimizationData._NuGetPackageFileName)"
+                                      DestinationFolder="$(_ManagedDependenciesDir)"
+                                      NewVersionSuffix="$(_OptimizedNuGetPackageVersionSuffix)"
+                                      Parts="%(_DependencyWithOptimizationData.DevEnvLib)"
+                                      ReplacementFiles="%(_DependencyWithOptimizationData._OptimizeAssembly)"
+                                      Condition="'@(_DependencyWithOptimizationData)' != ''">
+      <Output TaskParameter="NewPackage" ItemName="FileWrites" />
+    </Roslyn.Tools.ReplacePackageParts>
+
+    <!-- Copy unoptimized dependencies -->
+    <Copy SourceFiles="@(_DependencyWithoutOptimizationData->'%(_NuGetPackageDir)%(_NuGetPackageFileName)')" DestinationFolder="$(_ManagedDependenciesDir)">
+      <Output TaskParameter="CopiedFiles" ItemName="FileWrites"/>
+    </Copy>
  </Target>
 </Project>
\ No newline at end of file