Merge pull request #21146 from khyperia/refactor_linux_build

Move logic from the Makefile to either cibuild.sh or build/scripts

Makefile

-SHELL = /usr/bin/env bash
-OS_NAME = $(shell uname -s)
-BUILD_CONFIGURATION = Debug
-BINARIES_PATH = $(shell pwd)/Binaries
-SRC_PATH = $(shell pwd)/src
-BOOTSTRAP_PATH = $(BINARIES_PATH)/Bootstrap
-BUILD_LOG_PATH =
-HOME_DIR = $(shell cd ~ && pwd)
-DOTNET_VERSION = 1.0.1
-DOTNET = $(BINARIES_PATH)/dotnet-cli/dotnet
-TARGET_FX = netcoreapp1.1
-
-MSBUILD_ADDITIONALARGS := /v:m /fl /fileloggerparameters:Verbosity=normal /p:Configuration=$(BUILD_CONFIGURATION)
-
-ifeq ($(OS_NAME),Linux)
-	RUNTIME_ID := $(shell . /etc/os-release && echo $$ID.$$VERSION_ID)-x64
-else ifeq ($(OS_NAME),Darwin)
-	RUNTIME_ID := osx.10.12-x64
-endif
-
-ifneq ($(BUILD_LOG_PATH),)
-	MSBUILD_ADDITIONALARGS := $(MSBUILD_ADDITIONALARGS) /fileloggerparameters:LogFile=$(BUILD_LOG_PATH)
-endif
-
-ifeq ($(BOOTSTRAP),true)
-	MSBUILD_ARGS = $(MSBUILD_ADDITIONALARGS) /p:CscToolPath=$(BOOTSTRAP_PATH)/csc /p:CscToolExe=csc /p:VbcToolPath=$(BOOTSTRAP_PATH)/vbc /p:VbcToolExe=vbc
-else
-	MSBUILD_ARGS = $(MSBUILD_ADDITIONALARGS)
-endif
-
-BUILD_CMD = dotnet build $(MSBUILD_ARGS)
-
-.PHONY: all bootstrap test toolset
-
-all: restore
-	@export PATH="$(BINARIES_PATH)/dotnet-cli:$(PATH)" ; \
-	export HOME="$(HOME_DIR)" ; \
-	$(BUILD_CMD) CrossPlatform.sln
-
-bootstrap: restore
-	export HOME="$(HOME_DIR)" ; \
-	export PATH="$(BINARIES_PATH)/dotnet-cli:$(PATH)" ; \
-	$(BUILD_CMD) src/Compilers/CSharp/CscCore && \
-	$(BUILD_CMD) src/Compilers/VisualBasic/VbcCore && \
-	mkdir -p $(BOOTSTRAP_PATH)/csc && mkdir -p $(BOOTSTRAP_PATH)/vbc && \
-	dotnet publish -c $(BUILD_CONFIGURATION) -r $(RUNTIME_ID) src/Compilers/CSharp/CscCore -o $(BOOTSTRAP_PATH)/csc && \
-	dotnet publish -c $(BUILD_CONFIGURATION) -r $(RUNTIME_ID) src/Compilers/VisualBasic/VbcCore -o $(BOOTSTRAP_PATH)/vbc
-	rm -rf Binaries/$(BUILD_CONFIGURATION)
-
-test:
-	@export PATH="$(BINARIES_PATH)/dotnet-cli:$(PATH)" ; \
-	export HOME="$(HOME_DIR)" ; \
-	dotnet publish -r $(RUNTIME_ID) src/Test/DeployCoreClrTestRuntime -o $(BINARIES_PATH)/$(BUILD_CONFIGURATION)/CoreClrTest -p:RoslynRuntimeIdentifier=$(RUNTIME_ID) && \
-	build/scripts/tests.sh $(BUILD_CONFIGURATION)
-
-restore: $(DOTNET)
-	export PATH="$(BINARIES_PATH)/dotnet-cli:$(PATH)" ; \
-	./build/scripts/restore.sh
-
-$(DOTNET):
-	mkdir -p $(BINARIES_PATH) ; \
-	pushd $(BINARIES_PATH) ; \
-	curl -O https://raw.githubusercontent.com/dotnet/cli/rel/1.0.0/scripts/obtain/dotnet-install.sh && \
-	chmod +x dotnet-install.sh && \
-	./dotnet-install.sh --version "$(DOTNET_VERSION)" --install-dir "$(BINARIES_PATH)/dotnet-cli"
-
-
-clean:
-	@rm -rf Binaries

build/scripts/obtain_dotnet.sh

+#!/usr/bin/env bash
+# Copyright (c) .NET Foundation and contributors. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+# Source this script to ensure dotnet is installed and on the path.
+# If the FORCE_DOWNLOAD environment variable is set to "true", the system's dotnet install is ignored,
+# and dotnet is downloaded and installed locally.
+
+set -e
+set -u
+
+# check if `dotnet` is already on the PATH
+if command -v dotnet >/dev/null 2>&1
+then
+    if [[ "${FORCE_DOWNLOAD:-false}" != true ]]
+    then
+        exit 0
+    fi
+fi
+
+# This is a function to keep variable assignments out of the parent script (that is sourcing this file)
+install_dotnet () {
+    # Download and install `dotnet` locally
+    local DOTNET_VERSION=1.0.1
+    local THIS_DIR=$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+    local DOTNET_PATH=${THIS_DIR}/../../Binaries/dotnet-cli
+
+    if [[ ! -x "${DOTNET_PATH}/dotnet" ]]
+    then
+        echo "Downloading and installing .NET CLI version ${DOTNET_VERSION} to ${DOTNET_PATH}"
+        curl https://raw.githubusercontent.com/dotnet/cli/rel/${DOTNET_VERSION}/scripts/obtain/dotnet-install.sh | \
+            /usr/bin/env bash -s -- --version "${DOTNET_VERSION}" --install-dir "${DOTNET_PATH}"
+    else
+        echo "Skipping download of .NET CLI: Already installed at ${DOTNET_PATH}"
+    fi
+
+    export PATH=${DOTNET_PATH}:${PATH}
+}
+install_dotnet

build/scripts/tests.sh

 #!/usr/bin/env bash
+# Copyright (c) .NET Foundation and contributors. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
-BUILD_CONFIGURATION=$1
+set -e
+set -u
 
-# This function will update the PATH variable to put the desired
-# version of Mono ahead of the system one. 
+BUILD_CONFIGURATION=${1:-Debug}
 
-cd Binaries/$BUILD_CONFIGURATION/CoreClrTest
+THIS_DIR=$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+BINARIES_PATH=${THIS_DIR}/../../Binaries
+SRC_PATH=${THIS_DIR}/../../src
+TEST_DIR=${BINARIES_PATH}/${BUILD_CONFIGURATION}/CoreClrTest
 
-chmod +x ./corerun
+RUNTIME_ID=$(dotnet --info | awk '/RID:/{print $2;}')
+
+BUILD_ARGS="-c ${BUILD_CONFIGURATION} -r ${RUNTIME_ID} /consoleloggerparameters:Verbosity=minimal;summary /p:RoslynRuntimeIdentifier=${RUNTIME_ID}"
+dotnet publish ${SRC_PATH}/Test/DeployCoreClrTestRuntime -o ${TEST_DIR} ${BUILD_ARGS}
+
+cd ${TEST_DIR}
 
 mkdir -p xUnitResults
 
-./corerun ./xunit.console.netcore.exe *.UnitTests.dll -parallel all -xml xUnitResults/TestResults.xml
+dotnet exec ./xunit.console.netcore.exe *.UnitTests.dll -parallel all -xml xUnitResults/TestResults.xml
 
 if [ $? -ne 0 ]; then
     echo Unit test failed
     exit 1
 fi
-

cibuild.sh

 #!/usr/bin/env bash
+# Copyright (c) .NET Foundation and contributors. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
 set -e
+set -u
 
 usage()
 {
@@ -9,23 +13,22 @@ usage()
     echo "Options"
     echo "  --debug               Build Debug (default)"
     echo "  --release             Build Release"
+    echo "  --cleanrun            Clean the project before building"
     echo "  --skiptest            Do not run tests"
-    echo "  --skipcrossgen        Do not crossgen the bootstrapped compiler"
     echo "  --skipcommitprinting  Do not print commit information"
-    echo "  --nocache       Force download of toolsets"
 }
 
+THIS_DIR=$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+BINARIES_PATH=${THIS_DIR}/Binaries
+BOOTSTRAP_PATH=${BINARIES_PATH}/Bootstrap
+SRC_PATH=${THIS_DIR}/src
+BUILD_LOG_PATH=${BINARIES_PATH}/Build.log
+
 BUILD_CONFIGURATION=Debug
-USE_CACHE=true
+CLEAN_RUN=false
 SKIP_TESTS=false
-SKIP_CROSSGEN=false
 SKIP_COMMIT_PRINTING=false
 
-MAKE="make"
-if [[ $OSTYPE == *[Bb][Ss][Dd]* ]]; then
-    MAKE="gmake"
-fi
-
 # LTTNG is the logging infrastructure used by coreclr.  Need this variable set 
 # so it doesn't output warnings to the console.
 export LTTNG_HOME=$HOME
@@ -51,22 +54,23 @@ do
         BUILD_CONFIGURATION=Release
         shift 1
         ;;
-        --nocache)
-        USE_CACHE=false
+        --cleanrun)
+        CLEAN_RUN=true
         shift 1
         ;;
         --skiptests)
         SKIP_TESTS=true
         shift 1
         ;;
-        --skipcrossgen)
-        SKIP_CROSSGEN=true
-        shift 1
-        ;;
         --skipcommitprinting)
         SKIP_COMMIT_PRINTING=true
         shift 1
         ;;
+        --nocache)
+        # Temporarily ignore this argument until the change to netci.groovy gets merged.
+        # A follow-up PR will be made to remove this ignore.
+        shift 1
+        ;;
         *)
         usage 
         exit 1
@@ -74,8 +78,6 @@ do
     esac
 done
 
-MAKE_ARGS="BUILD_CONFIGURATION=$BUILD_CONFIGURATION SKIP_CROSSGEN=$SKIP_CROSSGEN"
-
 if [ "$CLEAN_RUN" == "true" ]; then
     echo Clean out the enlistment
     git clean -dxf . 
@@ -86,13 +88,33 @@ if [ "$SKIP_COMMIT_PRINTING" == "false" ]; then
     git show --no-patch --pretty=raw HEAD
 fi
 
-echo Building Bootstrap
-$MAKE bootstrap $MAKE_ARGS 
+# obtain_dotnet.sh puts the right dotnet on the PATH
+FORCE_DOWNLOAD=true
+source ${THIS_DIR}/build/scripts/obtain_dotnet.sh
 
-echo Building CrossPlatform.sln
-$MAKE all $MAKE_ARGS BOOTSTRAP=true BUILD_LOG_PATH=Binaries/Build.log
+RUNTIME_ID=$(dotnet --info | awk '/RID:/{print $2;}')
+echo "Using Runtime Identifier: ${RUNTIME_ID}"
 
-if [ "$SKIP_TESTS" == "false" ]; then
-    $MAKE test $MAKE_ARGS
-fi
+RESTORE_ARGS="-r ${RUNTIME_ID} -v Minimal --disable-parallel"
+echo "Restoring BaseToolset.csproj"
+dotnet restore ${RESTORE_ARGS} ${THIS_DIR}/build/ToolsetPackages/BaseToolset.csproj
+echo "Restoring CrossPlatform.sln"
+dotnet restore ${RESTORE_ARGS} ${THIS_DIR}/CrossPlatform.sln
 
+BUILD_ARGS="-c ${BUILD_CONFIGURATION} -r ${RUNTIME_ID} /nologo /consoleloggerparameters:Verbosity=minimal;summary /filelogger /fileloggerparameters:Verbosity=normal;logFile=${BUILD_LOG_PATH} /p:RoslynRuntimeIdentifier=${RUNTIME_ID}"
+
+echo "Building bootstrap CscCore"
+dotnet publish ${SRC_PATH}/Compilers/CSharp/CscCore -o ${BOOTSTRAP_PATH}/csc ${BUILD_ARGS}
+echo "Building bootstrap VbcCore"
+dotnet publish ${SRC_PATH}/Compilers/VisualBasic/VbcCore -o ${BOOTSTRAP_PATH}/vbc ${BUILD_ARGS}
+rm -rf ${BINARIES_PATH}/${BUILD_CONFIGURATION}
+BUILD_ARGS+=" /p:CscToolPath=${BOOTSTRAP_PATH}/csc /p:CscToolExe=csc /p:VbcToolPath=${BOOTSTRAP_PATH}/vbc /p:VbcToolExe=vbc"
+
+echo "Building CrossPlatform.sln"
+dotnet build ${THIS_DIR}/CrossPlatform.sln ${BUILD_ARGS}
+
+if [[ "${SKIP_TESTS}" == false ]]
+then
+    echo "Running tests"
+    ${THIS_DIR}/build/scripts/tests.sh ${BUILD_CONFIGURATION}
+fi

docs/infrastructure/cross-platform.md

@@ -4,26 +4,15 @@
 
 Linux and Mac support for developing Roslyn is very much a work in progress.  Not everything is supported at the moment and the steps detailed on this page will change very frequently.  If this is an area you are interested in then please check back frequently for updates.
 
-## Building using a pre-made toolset
+## Building
 
-Right now Roslyn builds on *nix using a mix of Mono and CoreCLR. Patching the right Mono version and acquiring all the tools
-can be very difficult, so we've saved pre-built versions on Azure.
+Build all cross-platform projects with: `dotnet build CrossPlatform.sln`.
 
-Running `make` should download all these toolset binaries and kick off a build using MSBuild running on Mono.
+If you do not have a system-wide `dotnet` install, you can obtain one with `./build/scripts/obtain_dotnet.sh`. This will install a compatible version of the CLI to `./Binaries/dotnet-cli` - add this to your PATH before trying to build `CrossPlatform.sln`. Alternatively, sourcing the script with `source ./build/scripts/obtain_dotnet.sh` will add it to your PATH for you.
 
 ## Using the compiler
 
-After building there should be at least two versions of `csc.exe` in your output directory.
-
-The first is in the `Binaries/Debug` directory. This is the "full .NET framework" version. That means it expects to run on a
-full .NET framework, like either the Windows .NET framework or Mono. You would run this like you run other mono programs, i.e.
-`mono csc.exe`.
-
-The second copy is in the `Binaries/Debug/csccore` directory. This is a version running directly on CoreCLR -- no Mono necessary.
-Just run `csc` in that directory. Note that this version includes a copy of CoreCLR in the output directory, so it is not portable.
-The version of CoreCLR copied is specific to whatever machine you built with, so if you're running OS X, this will only run on OS X.
-Similarly with Linux (and whatever distro you're using). 
-
+After building, there will be a `csc` in the `Binaries/Debug/Exes/CscCore` directory.
 
 ### Known issues when running `csc.exe`
 

docs/infrastructure/unix-toolset.md

-Building a new Unix toolset
-====
-This document describes building a new toolset for use in Mac or Linux.
-Because the toolsets contain various targets and reference assemblies that
-only exist on Windows, the toolsets currently must be built on Windows.
-
-### Building Roslyn Toolset
-The new *toolset name* will be chosen as one of the follownig:
-
-- Linux: roslyn.linux.`<version number>`
-- Mac: roslyn.mac.`<version number>`
-
-The value of *version number* will simply be the one number higher than the current version number of the toolset.  
-
-To build the toolset do the following:
-
-- If necessary, make modifications to the dependencies in the 
- `build/MSBuildToolset/project.json` file to bring in anything new.
-- Run the `build/MSBuildToolset/build-toolset.ps1` file.
-- The script produces two zip files in bin\Debug\netcoreapp1.0 subdirectory:
-    - Rename `osx.10.10-x64.zip` to roslyn.mac.`<version number>.zip`
-    - Rename `ubuntu.14.04-x64.zip` to roslyn.linux.`<version number>.zip`
-- Upload the files to the Azure in the dotnetci storage account in the roslyn container:
-
-```
-azcopy /Pattern:*.zip /Source:build\MSBuildToolset\bin\Debug\netcoreapp1.0 /Dest:https://dotnetci.blob.core.windows.net/roslyn /DestKey:<<key>>
-```
-
-- Send a PR to change [Makefile](https://github.com/dotnet/roslyn/blob/master/Makefile) to use the new toolset.  

netci.groovy

@@ -103,7 +103,7 @@ commitPullList.each { isPr ->
   def myJob = job(jobName) {
     description("Ubuntu 14.04 tests")
                   steps {
-                    shell("./cibuild.sh --nocache --debug")
+                    shell("./cibuild.sh --debug")
                   }
                 }
 
@@ -120,7 +120,7 @@ commitPullList.each { isPr ->
   def myJob = job(jobName) {
     description("Ubuntu 16.04 tests")
                   steps {
-                    shell("./cibuild.sh --nocache --debug")
+                    shell("./cibuild.sh --debug")
                   }
                 }
 
@@ -137,7 +137,7 @@ commitPullList.each { isPr ->
   def myJob = job(jobName) {
     description("Mac tests")
     steps {
-      shell("./cibuild.sh --nocache --debug")
+      shell("./cibuild.sh --debug")
     }
   }