Enable using Live-live builds of CoreCLR and CoreFX with Core-S… (dotnet/core-setup#8448)

* Enable using a live CoreCLR build when building core-setup by passing a path to a CoreCLR product directory via /p:CoreCLROverridePath.

* Enable using a live CoreFX build for Core-Setup restore.

* Grab ucrt redist from CoreCLR output directory.

* Fix overriding long-name DAC on CoreCLR-live build.

* Don't add non-existent doc files to the list of doc files to package.

* Add documentation about how to build with a local coreclr or corefx.

* PR Feedback.

* Extract out the overrides to a local targets file for the netcoreapp shared framework restore. Call out in the documentation of this build feature that it is only supported for the NETCoreApp shared framework.

* Update src/pkg/projects/netcoreapp/src/localnetcoreapp.override.targets
Co-Authored-By: NDavis Goodin <dagood@users.noreply.github.com>


Commit migrated from https://github.com/dotnet/core-setup/commit/b903a6e815dbcec7acfeef39db1e9189fef7ed69

docs/installer/building/local-coreclr-corefx.md

+# Build Core-Setup with a Local CoreCLR or CoreFX
+
+## Testing with private CoreCLR bits
+
+Generally the Core-Setup build system gets the CoreCLR from a NuGet package which gets pulled down and correctly copied to the various output directories by building `src\pkg\projects\netcoreapp\src\netcoreapp.depproj` which gets built as part of `build.cmd/sh`. For folks that want to do builds and test runs in corefx with a local private build of coreclr you can follow these steps:
+
+1. Build CoreCLR and note your output directory. Ex: `\coreclr\bin\Product\Windows_NT.x64.Release\` Note this will vary based on your OS/Architecture/Flavor and it is generally a good idea to use Release builds for CoreCLR when building Core-Setup and the OS and Architecture should match what you are building in Core-Setup.
+2. Build Core-Setup either passing in the `CoreCLROverridePath` property or setting it as an environment variable:
+
+```batch
+build.cmd /p:CoreCLROverridePath=d:\git\coreclr\bin\Product\Windows_NT.x64.Release
+```
+
+By convention the project will look for PDBs in a directory under `$(CoreCLROverridePath)/PDB` and if found will also copy them. If not found no PDBs will be copied. If you want to explicitly set the PDB path then you can pass `CoreCLRPDBOverridePath` property to that PDB directory.
+Once you have updated your CoreCLR you can run tests however you usually do and it should be using your copy of CoreCLR.
+If you prefer, you can use a Debug build of System.Private.CoreLib, but if you do you must also use a Debug build of the native portions of the runtime, e.g. coreclr.dll. Tests with a Debug runtime will execute much more slowly than with Release runtime bits. This override is only enabled for building the NETCoreApp shared framework.
+
+## Testing with private CoreFX bits
+
+Similarly to how Core-Setup consumes CoreCLR, Core-Setup also consumes CoreFX from a NuGet package which gets pulled down and correctly copied to the various output directories by building `src\pkg\projects\netcoreapp\src\netcoreapp.depproj` which gets built as part of `build.cmd/sh`. To test with a local private build of CoreFX, you can follow these steps:
+
+1. Build CoreFX and note your configuration (Ex. Debug, Checked, Release).
+2. Build Core-Setup either passing in the `CoreFXOverridePath` property or setting it as an environment variable (assuming your CoreFX repo is located at `d:\git\corefx`):
+
+```batch
+build.cmd /p:CoreFXOverridePath=d:\git\corefx\artifacts\packages\Debug
+```
+
+The Core-Setup build will resolve the correct libraries to use to build the NETCoreApp shared framework based on the restored packages and will locate the files in the output directory based on the nuspec file of the shared-framework package in the local CoreFX build. This override does not enable using a local CoreFX build when creating the WindowsDesktop shared framework.

src/installer/pkg/packaging-tools/framework.dependency.targets

@@ -251,10 +251,17 @@
       <_diaSymReaderPackageDir>$(PackagesDir)microsoft.diasymreader.native/$(MicrosoftDiaSymReaderNativePackageVersion)/</_diaSymReaderPackageDir>
     </PropertyGroup>
 
+    <PropertyGroup Condition="'$(TargetArchitecture)' == 'arm' AND '$(OS)' == 'Windows_NT'">
+      <_crossHostArch>x86</_crossHostArch>
+    </PropertyGroup> 
+    <PropertyGroup Condition="'$(TargetArchitecture)' == 'arm' AND '$(OS)' != 'Windows_NT'">
+      <_crossHostArch>x64</_crossHostArch>
+    </PropertyGroup> 
+    <PropertyGroup Condition="'$(TargetArchitecture)' == 'arm64'">
+      <_crossHostArch>x64</_crossHostArch>
+    </PropertyGroup> 
     <PropertyGroup>
-      <_crossDir Condition="'$(TargetArchitecture)' == 'arm' AND '$(OS)' == 'Windows_NT'">/x86_arm</_crossDir>
-      <_crossDir Condition="'$(TargetArchitecture)' == 'arm' AND '$(OS)' != 'Windows_NT'">/x64_arm</_crossDir>
-      <_crossDir Condition="'$(TargetArchitecture)' == 'arm64'">/x64_arm64</_crossDir>
+      <_crossDir Condition="'$(_crossHostArch)' != ''">/$(_crossHostArch)_$(TargetArchitecture)</_crossDir>
     </PropertyGroup>
 
     <ItemGroup>
@@ -303,7 +310,7 @@
     <PropertyGroup Condition="'@(_diaSymReaderAssembly)' != ''">
       <_diaSymReaderToolDir>%(_diaSymReaderAssembly.RootDir)%(_diaSymReaderAssembly.Directory)</_diaSymReaderToolDir>
     </PropertyGroup>
-
+    
     <!--
       DiaSymReader can't be built from source, so use an unrelated default directory in that case.
       This is used as the working directory for crossgen calls.
@@ -313,7 +320,7 @@
     </PropertyGroup>
 
     <ItemGroup>
-      <_requiredProperty Include="_coreLibDirectory;_crossGenPath;_jitPath;_fxLibDirectory;_windowsWinMDDirectory;_diaSymReaderToolDir" />
+      <_requiredProperty Include="_runtimeDirectory;_coreLibDirectory;_crossGenPath;_jitPath;_fxLibDirectory;_windowsWinMDDirectory;_diaSymReaderToolDir" />
     </ItemGroup>
 
     <Message Text="%(_requiredProperty.Identity): $(%(_requiredProperty.Identity))" />
@@ -404,7 +411,7 @@
           DependsOnTargets="
             GetCrossgenToolPaths;
             EnsureCrossGenIsExecutable"
-          Inputs="@(_filesToCrossGen)"
+          Inputs="@(_filesToCrossGen);$(_crossGenPath)"
           Outputs="%(_filesToCrossGen.CrossGenedPath)">
     <PropertyGroup>
       <_crossGenResponseFile>$(_crossGenIntermediatePath)%(_filesToCrossGen.FileName).rsp</_crossGenResponseFile>

src/installer/pkg/projects/netcoreapp/src/localnetcoreapp.override.targets

+<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <!-- Targets to enable building the NETCoreApp shared framework from a local CoreCLR and CoreFX build. -->
+  
+  <Target Name="OverrideRuntimeFilesFromPackageResolve" Condition="'$(CoreCLROverridePath)' != ''" BeforeTargets="GetFilesFromPackageResolve">
+    <Error Condition="!Exists('$(CoreCLROverridePath)')" Text="The path provided to CoreCLROverridePath ($(CoreCLROverridePath)) does not exist." />
+    <PropertyGroup>
+      <CoreCLRPDBOverridePath Condition="'$(CoreCLRPDBOverridePath)' == '' and Exists('$(CoreCLROverridePath)/PDB')">$(CoreCLROverridePath)/PDB</CoreCLRPDBOverridePath>
+    </PropertyGroup>
+    <ItemGroup>
+      <CoreCLRFiles Include="$(CoreCLROverridePath)/*.*" />
+      <CoreCLRFiles Include="$(CoreCLROverridePath)/Redist/**/*.dll" />
+      <CoreCLRFiles Condition="'$(CoreCLRPDBOverridePath)' != ''"
+        Include="$(CoreCLRPDBOverridePath)/*.pdb;$(CoreCLRPDBOverridePath)/*.dbg;$(CoreCLRPDBOverridePath)/*.dwarf" />
+
+      <OverriddenRuntimeFiles Include="@(ReferenceCopyLocalPaths)" Condition="'@(CoreCLRFiles->'%(FileName)%(Extension)')' == '%(FileName)%(Extension)'">
+        <CoreCLRFile>@(CoreCLRFiles)</CoreCLRFile>
+      </OverriddenRuntimeFiles>
+
+      <LongNameDacFile Include="@(ReferenceCopyLocalPaths)" Condition="$([System.String]::Copy('%(FileName)').StartsWith('mscordaccore_'))">
+        <TargetPath>%(PathInPackage)</TargetPath>
+      </LongNameDacFile>
+
+      <ShortNameDacFileOverride Include="@(CoreCLRFiles)" Condition="'%(FileName)%(Extension)' == 'mscordaccore.dll'" />
+
+      <OverriddenRuntimeFiles Include="@(LongNameDacFile)">
+        <CoreCLRFile>@(ShortNameDacFileOverride)</CoreCLRFile>
+      </OverriddenRuntimeFiles>
+
+      <OverriddenRuntimeFiles>
+        <IsNative Condition="$([System.String]::new('%(Identity)').ToLowerInvariant().Replace('\', '/').Contains('/native/'))">true</IsNative>
+      </OverriddenRuntimeFiles>
+
+      <ReferenceCopyLocalPaths Remove="@(OverriddenRuntimeFiles)" />
+      <ReferenceCopyLocalPaths Include="@(OverriddenRuntimeFiles->Metadata('CoreCLRFile'))" />
+    </ItemGroup>
+
+    <Error Condition="'@(CoreCLRFiles)' == ''" Text="The path provided to CoreCLROverridePath ($(CoreCLROverridePath)) does not contain any files." />
+  </Target>
+
+  <Target Name="OverrideFrameworkFilesFromPackageResolve" Condition="'$(CoreFXOverridePath)' != ''" BeforeTargets="GetFilesFromPackageResolve">
+    <Error Condition="!Exists('$(CoreFXOverridePath)')" Text="The path provided to CoreFXOverridePath ($(CoreFXOverridePath)) does not exist." />
+
+    <PropertyGroup>
+      <CoreFXSharedFrameworkPackageSpec>$(CoreFXOverridePath)/specs/$(MicrosoftPrivateCoreFxNETCoreAppPackage).nuspec</CoreFXSharedFrameworkPackageSpec>
+      <CoreFXSharedFrameworkImplementationPackageSpec Condition="'$(PackageRID)' != ''">$(CoreFXOverridePath)/specs/runtime.$(PackageRID).$(MicrosoftPrivateCoreFxNETCoreAppPackage).nuspec</CoreFXSharedFrameworkImplementationPackageSpec>
+    </PropertyGroup>
+    
+    <Error Condition="!Exists('$(CoreFXSharedFrameworkPackageSpec)')" Text="The nuspec for the reference Microsoft.Private.CoreFx.NETCoreApp package does not exist." />
+    <Message Condition="'$(CoreFXSharedFrameworkImplementationPackageSpec)' != '' And !Exists('$(CoreFXSharedFrameworkImplementationPackageSpec)')"
+      Text="The nuspec for the implementation runtime.$(PackageRID).Microsoft.Private.CoreFx.NETCoreApp package does not exist. Falling back to packaged CoreFX." />
+
+    <XmlPeek
+      Namespaces="&lt;Namespace Prefix='nuget' Uri='http://schemas.microsoft.com/packaging/2013/01/nuspec.xsd' /&gt;"
+      Query="nuget:package/nuget:files/nuget:file/@src"
+      XmlInputPath="$(CoreFXSharedFrameworkPackageSpec)">
+      <Output TaskParameter="Result" ItemName="CoreFXReferenceItems" />
+    </XmlPeek>
+
+    <XmlPeek
+      Condition="'$(CoreFXSharedFrameworkImplementationPackageSpec)' != '' And Exists($(CoreFXSharedFrameworkImplementationPackageSpec))"
+      Namespaces="&lt;Namespace Prefix='nuget' Uri='http://schemas.microsoft.com/packaging/2013/01/nuspec.xsd' /&gt;"
+      Query="nuget:package/nuget:files/nuget:file/@src"
+      XmlInputPath="$(CoreFXSharedFrameworkImplementationPackageSpec)">
+      <Output TaskParameter="Result" ItemName="CoreFXReferenceCopyLocalItems" />
+    </XmlPeek>
+    
+    <ItemGroup>
+      <CoreFXReferenceItems NuGetPackageId="Microsoft.Private.CoreFx.NETCoreApp" />
+      <CoreFXReferenceCopyLocalItems NuGetPackageId="runtime.$(PackageRID).Microsoft.Private.CoreFx.NETCoreApp" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <OverriddenFrameworkReferenceFiles
+        Include="@(Reference)"
+        Condition="
+          '@(CoreFXReferenceItems->'%(FileName)%(Extension)')' == '%(FileName)%(Extension)' And
+          '@(CoreFXReferenceItems->'%(NuGetPackageId)')' == '%(NuGetPackageId)'">
+        <CoreFXFile>@(CoreFXReferenceItems)</CoreFXFile>
+      </OverriddenFrameworkReferenceFiles>
+
+      <Reference Remove="@(OverriddenFrameworkReferenceFiles)" />
+      <Reference Include="@(OverriddenFrameworkReferenceFiles->Metadata('CoreFXFile'))" />
+
+      <OverriddenFrameworkImplementationFiles
+        Include="@(ReferenceCopyLocalPaths)"
+        Condition="
+          '@(CoreFXReferenceCopyLocalItems->'%(FileName)%(Extension)')' == '%(FileName)%(Extension)' And
+          '@(CoreFXReferenceCopyLocalItems->'%(NuGetPackageId)')' == '%(NuGetPackageId)'">
+        <CoreFXFile>@(CoreFXReferenceCopyLocalItems)</CoreFXFile>
+      </OverriddenFrameworkImplementationFiles>
+
+      <OverriddenFrameworkImplementationFiles>
+        <IsNative Condition="$([System.String]::new('%(Identity)').ToLowerInvariant().Replace('\', '/').Contains('/native/'))">true</IsNative>
+      </OverriddenFrameworkImplementationFiles>
+
+      <ReferenceCopyLocalPaths Remove="@(OverriddenFrameworkImplementationFiles)" />
+      <ReferenceCopyLocalPaths Include="@(OverriddenFrameworkImplementationFiles->Metadata('CoreFXFile'))" />
+    </ItemGroup>
+
+    <ItemGroup Condition="'$(PackageTargetRuntime)' == ''">
+      <_coreFXOverrideDocFiles
+        Condition="'$(CoreFXOverridePath)' != '' And '%(NuGetPackageId)' == '$(MicrosoftPrivateCoreFxNETCoreAppPackage)'"
+        Include="@(OverriddenFrameworkReferenceFiles->'$(CoreFXOverridePath)/../../bin/docs/%(FileName).xml')" />
+      <_docFilesToPackage Include="@(_coreFXOverrideDocFiles)" Condition="Exists('%(Identity)')" />
+    </ItemGroup>
+  </Target>
+
+  <Target Name="OverrideCrossgenToolPaths" DependsOnTargets="GetCorePackagePaths" AfterTargets="GetCrossgenToolPaths" BeforeTargets="CreateCrossGenImages">  
+    <PropertyGroup Condition="'$(CoreCLROverridePath)' != ''">
+      <_runtimeDirectory>$(CoreCLROverridePath)</_runtimeDirectory>
+      <_crossgenPath>$(CoreCLROverridePath)/crossgen$(ApplicationFileExtension)</_crossgenPath>
+      <_coreLibDirectory>$(CoreCLROverridePath)</_coreLibDirectory>
+      <_jitPath>$(CoreClrOverridePath)/$(_crossHostArch)/$(LibraryFilePrefix)clrjit$(LibraryFileExtension)</_jitPath>
+    </PropertyGroup>
+
+  </Target>
+
+</Project>

src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj

@@ -77,5 +77,6 @@
       </FilesToPackage>
     </ItemGroup>
   </Target>
-
+  
+  <Import Project="localnetcoreapp.override.targets" />
 </Project>