Update the compiler in the Linux/Mac toolset

Updates the compiler used in the Linux and Mac toolsets and also
eliminates the requirement to download the toolset as a zip and instead
restores the necessary components over NuGet.

Makefile

@@ -3,24 +3,23 @@ OS_NAME = $(shell uname -s)
 BUILD_CONFIGURATION = Debug
 BINARIES_PATH = $(shell pwd)/Binaries
 SCRIPTS_PATH = $(shell pwd)/build/scripts
+SRC_PATH = $(shell pwd)/src
+TOOLSET_SRC_PATH = $(shell pwd)/build/MSBuildToolset
 TOOLSET_PATH = $(BINARIES_PATH)/toolset
-RESTORE_SEMAPHORE_PATH = $(TOOLSET_PATH)/restore.semaphore
+RESTORE_SEMAPHORE_PATH = $(BINARIES_PATH)/restore.semaphore
 BOOTSTRAP_PATH = $(BINARIES_PATH)/Bootstrap
 BUILD_LOG_PATH =
 HOME_DIR = $(shell cd ~ && pwd)
-DOTNET_VERSION = 1.0.0-preview2-002911
 NUGET_VERSION = 3.5.0-beta2
 NUGET_EXE = $(shell pwd)/nuget.exe
 
-MSBUILD_ADDITIONALARGS := /v:m /fl /fileloggerparameters:Verbosity=normal /p:Configuration=$(BUILD_CONFIGURATION)
+MSBUILD_ADDITIONALARGS := /fl /fileloggerparameters:Verbosity=normal /p:Configuration=$(BUILD_CONFIGURATION)
 
 ifeq ($(OS_NAME),Linux)
 	MSBUILD_ADDITIONALARGS := $(MSBUILD_ADDITIONALARGS) /p:BaseNuGetRuntimeIdentifier=ubuntu.14.04
-	ROSLYN_TOOLSET_NAME = roslyn.linux.8
 	DOTNET_PLATFORM = ubuntu-x64
 else ifeq ($(OS_NAME),Darwin)
 	MSBUILD_ADDITIONALARGS := $(MSBUILD_ADDITIONALARGS) /p:BaseNuGetRuntimeIdentifier=osx.10.10
-	ROSLYN_TOOLSET_NAME = roslyn.mac.8
 	DOTNET_PLATFORM = osx-x64
 endif
 
@@ -28,25 +27,21 @@ ifneq ($(BUILD_LOG_PATH),)
 	MSBUILD_ADDITIONALARGS := $(MSBUILD_ADDITIONALARGS) /fileloggerparameters:LogFile=$(BUILD_LOG_PATH)
 endif
 
-ROSLYN_TOOLSET_PATH = $(TOOLSET_PATH)/$(ROSLYN_TOOLSET_NAME)
-
 ifeq ($(BOOTSTRAP),true)
 	MSBUILD_ARGS = $(MSBUILD_ADDITIONALARGS) /p:CscToolPath=$(BOOTSTRAP_PATH) /p:CscToolExe=csc /p:VbcToolPath=$(BOOTSTRAP_PATH) /p:VbcToolExe=vbc
 else
 	MSBUILD_ARGS = $(MSBUILD_ADDITIONALARGS) /p:CscToolExe=csc /p:VbcToolExe=vbc
 endif
 
-MSBUILD_CMD = $(ROSLYN_TOOLSET_PATH)/corerun $(ROSLYN_TOOLSET_PATH)/MSBuild.exe $(MSBUILD_ARGS)
+MSBUILD_CMD = $(TOOLSET_PATH)/corerun $(TOOLSET_PATH)/MSBuild.dll $(MSBUILD_ARGS)
 
 .PHONY: all bootstrap test restore toolset nuget
 
-all: $(ROSLYN_TOOLSET_PATH) $(RESTORE_SEMAPHORE_PATH)
-	export ReferenceAssemblyRoot=$(ROSLYN_TOOLSET_PATH)/reference-assemblies/Framework ; \
+all: $(RESTORE_SEMAPHORE_PATH)
 	export HOME=$(HOME_DIR) ; \
 	$(MSBUILD_CMD) CrossPlatform.sln
 
-bootstrap: $(ROSLYN_TOOLSET_PATH) $(RESTORE_SEMAPHORE_PATH)
-	export ReferenceAssemblyRoot=$(ROSLYN_TOOLSET_PATH)/reference-assemblies/Framework ; \
+bootstrap: $(TOOLSET_PATH) $(RESTORE_SEMAPHORE_PATH)
 	export HOME=$(HOME_DIR) ; \
 	$(MSBUILD_CMD) src/Compilers/CSharp/CscCore/CscCore.csproj && \
 	$(MSBUILD_CMD) src/Compilers/VisualBasic/VbcCore/VbcCore.csproj && \
@@ -61,8 +56,8 @@ test:
 
 restore: $(NUGET_EXE) $(RESTORE_SEMAPHORE_PATH)
 
-$(RESTORE_SEMAPHORE_PATH): $(ROSLYN_TOOLSET_PATH)
-	@build/scripts/restore.sh $(ROSLYN_TOOLSET_PATH) $(NUGET_EXE) && \
+$(RESTORE_SEMAPHORE_PATH): $(TOOLSET_PATH)
+	@build/scripts/restore.sh $(TOOLSET_PATH) $(NUGET_EXE) && \
 	touch $(RESTORE_SEMAPHORE_PATH)
 
 $(NUGET_EXE):
@@ -76,19 +71,8 @@ clean:
 clean_toolset:
 	@rm -rf $(TOOLSET_PATH)
 
-toolset: $(ROSLYN_TOOLSET_PATH)
-
-$(ROSLYN_TOOLSET_PATH): | $(TOOLSET_PATH)
-	@pushd $(TOOLSET_PATH) ; \
-	curl -O https://dotnetci.blob.core.windows.net/roslyn/$(ROSLYN_TOOLSET_NAME).zip && \
-	curl -O https://dotnetcli.blob.core.windows.net/dotnet/preview/Binaries/$(DOTNET_VERSION)/dotnet-dev-$(DOTNET_PLATFORM).$(DOTNET_VERSION).tar.gz && \
-	mkdir -p $(ROSLYN_TOOLSET_NAME)/dotnet-cli && \
-	$(SCRIPTS_PATH)/unzip.sh $(ROSLYN_TOOLSET_NAME).zip $(ROSLYN_TOOLSET_NAME) && \
-	tar -zxf dotnet-dev-$(DOTNET_PLATFORM).$(DOTNET_VERSION).tar.gz -C $(ROSLYN_TOOLSET_NAME)/dotnet-cli && \
-	chmod +x $(ROSLYN_TOOLSET_NAME)/corerun && \
-	chmod +x $(ROSLYN_TOOLSET_NAME)/RoslynRestore && \
-	chmod +x $(ROSLYN_TOOLSET_NAME)/csc && \
-	chmod +x $(ROSLYN_TOOLSET_NAME)/vbc
+toolset: $(TOOLSET_PATH)
 
 $(TOOLSET_PATH):
-	mkdir -p $(TOOLSET_PATH)
+	pushd $(TOOLSET_SRC_PATH) ; \
+	dotnet publish -o $(TOOLSET_PATH)

build/MSBuildToolset/Program.cs

@@ -29,7 +29,7 @@ public static int Main(string[] args)
                 .ToArray();
 
             var nugetExeDir = Path.GetDirectoryName(nugetExePath);
-            string msbuildPath = Path.Combine(AppContext.BaseDirectory, "MSBuild.exe");
+            string msbuildPath = Path.Combine(AppContext.BaseDirectory, "MSBuild.dll");
             string entryPointTargetPath = Path.Combine(nugetExeDir,
                 "build/Targets/GetProjectsReferencingProjectJsonFilesEntryPoint.targets");
             string afterBuildsTargetPath = Path.Combine(nugetExeDir,

build/MSBuildToolset/csc

+#!/bin/sh
+
+THISDIR=$(dirname $0)
+
+chmod +x $THISDIR/corerun 2>/dev/null
+$THISDIR/corerun $THISDIR/csc.exe "$@"

build/MSBuildToolset/project.json

 {
   "version": "1.0.0-*",
-  "compilationOptions": {
+  "buildOptions": {
     "emitEntryPoint": true,
     "outputName": "RoslynRestore"
   },
+  "publishOptions": {
+      "include": [ "csc", "vbc" ]
+  },
   "dependencies": {
-    "Microsoft.Build": "14.3.0",
-    "Microsoft.Build.Tasks.Core": "14.3.0",
+    "Microsoft.Build": "15.1.0-preview-000370-00",
+    "Microsoft.Build.Runtime": "15.1.0-preview-000370-00",
+    "Microsoft.Build.Tasks.Core": "15.1.0-preview-000370-00",
+    "Microsoft.Net.Compilers.netcore": "2.0.0-rc2-61102-09",
+    "Microsoft.NuGet.Build.Tasks": "0.1.0",
+    "Microsoft.CodeAnalysis.Build.Tasks": "2.0.0-rc2-61102-09",
     "Microsoft.NETCore.TestHost": "1.0.0",
-    "Microsoft.NETCore.App": "1.0.0-rc3-004338",
+    "Microsoft.NETCore.App": "1.1.0",
+    "Microsoft.Portable.Targets": "0.1.1-dev",
     "Newtonsoft.Json": "8.0.3"
   },
   "frameworks": {
@@ -21,6 +29,6 @@
   },
   "runtimes": {
     "ubuntu.14.04-x64": {},
-    "osx.10.10-x64": {}
+    "osx.10.11-x64": {}
   }
 }

build/MSBuildToolset/vbc

+#!/bin/sh
+
+THISDIR=$(dirname $0)
+
+chmod +x $THISDIR/corerun 2>/dev/null
+$THISDIR/corerun $THISDIR/vbc.exe "$@"
+

build/Targets/DisableStandardFrameworkResolution.targets

+<!-- Copyright (c)  Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information. -->
+<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+
+  <Target Name="GetReferenceAssemblyPaths" />
+  <Target Name="GetFrameworkPaths" />
+
+  <PropertyGroup>
+    <_TargetFrameworkDirectories />
+    <FrameworkPathOverride />
+    <TargetFrameworkDirectory />
+
+    <!-- all references (even the StdLib) come from packages -->
+    <NoStdLib>true</NoStdLib>
+  </PropertyGroup>
+
+</Project>

build/Targets/Imports.targets

@@ -149,6 +149,7 @@
           <PropertyGroup>
             <AssemblyOriginatorKeyFile>$(MSBuildThisFileDirectory)..\Strong Name Keys\RoslynInternalKey.Private.snk</AssemblyOriginatorKeyFile>
             <DelaySign>false</DelaySign>
+            <PublicSign>true</PublicSign>
             <PublicKey>$(RoslynInternalKey)</PublicKey>
             <PublicKeyToken>fc793a00266884fb</PublicKeyToken>
           </PropertyGroup>
@@ -200,11 +201,16 @@
 
   <Import Project="$(MSBuildTargetsFilePath)" />
 
-  <!-- On Mono on *nix the NuGet targets aren't imported by default, so we do that here -->
-  <Import Project="$(MSBuildExtensionsPath)\Microsoft\NuGet\Microsoft.NuGet.targets" Condition="'$(OS)' != 'Windows_NT'" />
+  <!-- On *nix the NuGet targets aren't imported by default, so we do that here -->
+  <Import Project="$(MSBuildBinPath)\Microsoft.NuGet.targets" Condition="'$(OS)' != 'Windows_NT'" />
+
+  <!-- The portable v5.0 framework is delivered over NuGet, so std resolution should be disabled -->
+  <Import Project="DisableStandardFrameworkResolution.targets"
+          Condition="'$(TargetFrameworkIdentifier)' == '.NETPortable' AND '$(TargetFrameworkVersion)' == 'v5.0'" />
 
   <Choose>
     <When Condition="'$(TargetFrameworkIdentifier)' == '.NETPortable' AND '$(TargetFrameworkVersion)' == 'v5.0'">
+
       <!-- Treat portable exes as CoreClr-targeting-exes -->
       <PropertyGroup Condition="'$(OutputType)' == 'Exe'">
         <NuGetTargetMoniker>.NETCoreApp,Version=v1.0</NuGetTargetMoniker>

build/Targets/Settings.props

@@ -10,6 +10,7 @@
     <NuGetPackageRoot Condition="'$(NuGetPackageRoot)' == ''">$(NUGET_PACKAGES)</NuGetPackageRoot> <!-- Respect environment variable if set -->
     <NuGetPackageRoot Condition="'$(NuGetPackageRoot)' == '' and '$(OS)' == 'Windows_NT'">$(UserProfile)\.nuget\packages</NuGetPackageRoot>
     <NuGetPackageRoot Condition="'$(NuGetPackageRoot)' == '' and '$(OS)' != 'Windows_NT'">$(HOME)\.nuget\packages</NuGetPackageRoot>
+    <NuGetPackagesDirectory>$(NuGetPackageRoot)</NuGetPackagesDirectory>
     <ToolsetCompilerPackageName>Microsoft.Net.Compilers</ToolsetCompilerPackageName>
     <ToolsetCompilerPackageVersion>2.0.0-rc2-61102-09</ToolsetCompilerPackageVersion>
     <RoslynDiagnosticsNugetPackageVersion>1.2.0-beta2</RoslynDiagnosticsNugetPackageVersion>
@@ -69,14 +70,6 @@
     <!-- Point to the reference assemblies on unix -->
     <TargetFrameworkRootPath>$(MSBuildBinPath)\reference-assemblies\Framework</TargetFrameworkRootPath>
 
-    <!-- The /publicsign argument is required for the compiler, but the current MSBuild
-         build task can't be redirected and even if it could the new build task is not
-         built against the xplat MSBuild references so it can't be loaded properly. Providing
-         a response file works around the problem by directly adding the public sign argument
-         to all unix compilations. This shouldn't present a problem as it's impossible to
-         fully sign on unix at the moment anyway. Tracked by #7756. -->
-    <CompilerResponseFile>$(MSBuildThisFileDirectory)..\unix\extra_unix_args.rsp</CompilerResponseFile>
-
     <DebugType>portable</DebugType>
   </PropertyGroup>
 
@@ -250,6 +243,6 @@
 
   <ImportGroup Label="WindowsImports" Condition="'$(OS)' != 'Windows_NT'">
     <!-- NuGet props aren't imported by default on *nix so we do that here -->
-    <Import Project="$(MSBuildExtensionsPath)\Microsoft\NuGet\Microsoft.NuGet.props" Condition="'$(OS)' != 'Windows_NT'" />
+    <Import Project="$(MSBuildBinPath)\Microsoft.NuGet.props" Condition="'$(OS)' != 'Windows_NT'" />
   </ImportGroup>
 </Project>

build/scripts/restore.sh

@@ -9,8 +9,8 @@ export HOME=$(cd ~ && pwd)
 
 echo "Restoring toolset packages"
 
-$DOTNET_PATH restore -v Minimal --disable-parallel $(pwd)/build/ToolsetPackages/project.json
+dotnet restore -v Minimal --disable-parallel $(pwd)/build/ToolsetPackages/project.json
 
 echo "Restore CrossPlatform.sln"
 
-$ROSLYN_TOOLSET_PATH/RoslynRestore $(pwd)/CrossPlatform.sln $NUGET_EXE $DOTNET_PATH
+$ROSLYN_TOOLSET_PATH/RoslynRestore $(pwd)/CrossPlatform.sln $NUGET_EXE $(which dotnet)