[main] Update dependencies from dotnet/arcade (#11693)

* Update dependencies from https://github.com/dotnet/arcade build 20210611.3

Microsoft.DotNet.Arcade.Sdk
 From Version 6.0.0-beta.21303.2 -> To Version 6.0.0-beta.21311.3

* Update dependencies from https://github.com/dotnet/arcade build 20210614.1

Microsoft.DotNet.Arcade.Sdk
 From Version 6.0.0-beta.21311.3 -> To Version 6.0.0-beta.21314.1

* Update dependencies from https://github.com/dotnet/arcade build 20210616.3

Microsoft.DotNet.Arcade.Sdk
 From Version 6.0.0-beta.21314.1 -> To Version 6.0.0-beta.21316.3

* Update dependencies from https://github.com/dotnet/arcade build 20210617.1

Microsoft.DotNet.Arcade.Sdk
 From Version 6.0.0-beta.21316.3 -> To Version 6.0.0-beta.21317.1

* Update dependencies from https://github.com/dotnet/arcade build 20210618.2

Microsoft.DotNet.Arcade.Sdk
 From Version 6.0.0-beta.21303.2 -> To Version 6.0.0-beta.21318.2

* Update dependencies from https://github.com/dotnet/arcade build 20210619.2

Microsoft.DotNet.Arcade.Sdk
 From Version 6.0.0-beta.21303.2 -> To Version 6.0.0-beta.21319.2

* retain stable sdk
Co-authored-by: Ndotnet-maestro[bot] <dotnet-maestro[bot]@users.noreply.github.com>
Co-authored-by: NBrett V. Forsgren <brettfo@microsoft.com>

eng/Version.Details.xml

@@ -8,9 +8,9 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21303.2">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21319.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>78da7776965b428ff31da8f1ff2cb073506212b7</Sha>
+      <Sha>a3377cccde8639089f99107e2ba5df2c8cbe6394</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
   </ToolsetDependencies>

eng/common/dotnet-install.sh

@@ -70,7 +70,7 @@ case $cpuname in
     ;;
 esac
 
-dotnetRoot="$repo_root/.dotnet"
+dotnetRoot="${repo_root}.dotnet"
 if [[ $architecture != "" ]] && [[ $architecture != $buildarch ]]; then
   dotnetRoot="$dotnetRoot/$architecture"
 fi

eng/common/generate-locproject.ps1

@@ -25,8 +25,15 @@ Push-Location "$SourcesDirectory" # push location for Resolve-Path -Relative to
 
 # Template files
 $jsonFiles = @()
-$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\en\..+\.json" } # .NET templating pattern
-$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern
+$jsonTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\.+\.en\.json" } # .NET templating pattern
+$jsonTemplateFiles | ForEach-Object {
+    $null = $_.Name -Match "(.+)\.[\w-]+\.json" # matches '[filename].[langcode].json
+    
+    $destinationFile = "$($_.Directory.FullName)\$($Matches.1).json"
+    $jsonFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
+}
+
+$jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern
 
 $xlfFiles = @()
 
@@ -44,7 +51,7 @@ $langXlfFiles | ForEach-Object {
     $xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
 }
 
-$locFiles = $jsonFiles + $xlfFiles
+$locFiles = $jsonFiles + $jsonWinformsTemplateFiles + $xlfFiles
 
 $locJson = @{
     Projects = @(

eng/common/internal-feed-operations.ps1

@@ -45,11 +45,11 @@ function SetupCredProvider {
   # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable 
   # feeds successfully
 
-  $nugetConfigPath = "$RepoRoot\NuGet.config"
+  $nugetConfigPath = Join-Path $RepoRoot "NuGet.config"
 
   if (-Not (Test-Path -Path $nugetConfigPath)) {
     Write-PipelineTelemetryError -Category 'Build' -Message 'NuGet.config file not found in repo root!'
-    ExitWithExitCode 1  
+    ExitWithExitCode 1
   }
   
   $endpoints = New-Object System.Collections.ArrayList
@@ -85,7 +85,7 @@ function SetupCredProvider {
 
 #Workaround for https://github.com/microsoft/msbuild/issues/4430
 function InstallDotNetSdkAndRestoreArcade {
-  $dotnetTempDir = "$RepoRoot\dotnet"
+  $dotnetTempDir = Join-Path $RepoRoot "dotnet"
   $dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*)
   $dotnet = "$dotnetTempDir\dotnet.exe"
   $restoreProjPath = "$PSScriptRoot\restore.proj"

eng/common/internal-feed-operations.sh

@@ -39,7 +39,7 @@ function SetupCredProvider {
   # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable 
   # feeds successfully
 
-  local nugetConfigPath="$repo_root/NuGet.config"
+  local nugetConfigPath="{$repo_root}NuGet.config"
 
   if [ ! "$nugetConfigPath" ]; then
     Write-PipelineTelemetryError -category 'Build' "NuGet.config file not found in repo's root!"

eng/common/sdk-task.ps1

@@ -34,7 +34,7 @@ function Print-Usage() {
 function Build([string]$target) {
   $logSuffix = if ($target -eq 'Execute') { '' } else { ".$target" }
   $log = Join-Path $LogDir "$task$logSuffix.binlog"
-  $outputPath = Join-Path $ToolsetDir "$task\\"
+  $outputPath = Join-Path $ToolsetDir "$task\"
 
   MSBuild $taskProject `
     /bl:$log `
@@ -64,7 +64,7 @@ try {
       $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty
     }
     if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
-      $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.8.0-preview3" -MemberType NoteProperty
+      $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.10.0-preview2" -MemberType NoteProperty
     }
     if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") {
         $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true

eng/common/sdl/execute-all-sdl-tools.ps1

@@ -32,7 +32,7 @@ try {
   $ErrorActionPreference = 'Stop'
   Set-StrictMode -Version 2.0
   $disableConfigureToolsetImport = $true
-  $LASTEXITCODE = 0
+  $global:LASTEXITCODE = 0
 
   # `tools.ps1` checks $ci to perform some actions. Since the SDL
   # scripts don't necessarily execute in the same agent that run the

eng/common/sdl/init-sdl.ps1

@@ -10,7 +10,7 @@ Param(
 $ErrorActionPreference = 'Stop'
 Set-StrictMode -Version 2.0
 $disableConfigureToolsetImport = $true
-$LASTEXITCODE = 0
+$global:LASTEXITCODE = 0
 
 # `tools.ps1` checks $ci to perform some actions. Since the SDL
 # scripts don't necessarily execute in the same agent that run the

eng/common/sdl/run-sdl.ps1

@@ -13,7 +13,7 @@ Param(
 $ErrorActionPreference = 'Stop'
 Set-StrictMode -Version 2.0
 $disableConfigureToolsetImport = $true
-$LASTEXITCODE = 0
+$global:LASTEXITCODE = 0
 
 try {
   # `tools.ps1` checks $ci to perform some actions. Since the SDL

eng/common/templates/job/onelocbuild.yml

@@ -18,6 +18,9 @@ parameters:
   LclSource: lclFilesInRepo
   LclPackageId: ''
   RepoType: gitHub
+  GitHubOrg: dotnet
+  MirrorRepo: ''
+  MirrorBranch: main
   condition: ''
 
 jobs:
@@ -66,6 +69,11 @@ jobs:
         ${{ if eq(parameters.RepoType, 'gitHub') }}:
           repoType: ${{ parameters.RepoType }}
           gitHubPatVariable: "${{ parameters.GithubPat }}"
+        ${{ if ne(parameters.MirrorRepo, '') }}:
+          isMirrorRepoSelected: true
+          gitHubOrganization: ${{ parameters.GitHubOrg }}
+          mirrorRepo: ${{ parameters.MirrorRepo }}
+          mirrorBranch: ${{ parameters.MirrorBranch }}
       condition: ${{ parameters.condition }}
 
     - task: PublishBuildArtifacts@1

eng/common/templates/job/source-index-stage1.yml

 parameters:
   runAsPublic: false
-  sourceIndexPackageVersion: 1.0.1-20210421.1
+  sourceIndexPackageVersion: 1.0.1-20210614.1
   sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
   sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
   preSteps: []
   binlogPath: artifacts/log/Debug/Build.binlog
   pool:
     vmImage: vs2017-win2016
+  condition: ''
+  dependsOn: ''
 
 jobs:
 - job: SourceIndexStage1
+  dependsOn: ${{ parameters.dependsOn }}
+  condition: ${{ parameters.condition }}
   variables:
   - name: SourceIndexPackageVersion
     value: ${{ parameters.sourceIndexPackageVersion }}

eng/common/tools.ps1

@@ -193,38 +193,42 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) {
   return $global:_DotNetInstallDir = $dotnetRoot
 }
 
+function Retry($downloadBlock, $maxRetries = 5) {
+  $retries = 1
+
+  while($true) {
+    try {
+      & $downloadBlock
+      break
+    }
+    catch {
+      Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
+    }
+
+    if (++$retries -le $maxRetries) {
+      $delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff
+      Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)."
+      Start-Sleep -Seconds $delayInSeconds
+    }
+    else {
+      Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to download file in $maxRetries attempts."
+      break
+    }
+
+  }
+}
+
 function GetDotNetInstallScript([string] $dotnetRoot) {
   $installScript = Join-Path $dotnetRoot 'dotnet-install.ps1'
   if (!(Test-Path $installScript)) {
     Create-Directory $dotnetRoot
     $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
-
-    $maxRetries = 5
-    $retries = 1
-
     $uri = "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1"
 
-    while($true) {
-      try {
-        Write-Host "GET $uri"
-        Invoke-WebRequest $uri -OutFile $installScript
-        break
-      }
-      catch {
-        Write-Host "Failed to download '$uri'"
-        Write-Error $_.Exception.Message -ErrorAction Continue
-      }
-
-      if (++$retries -le $maxRetries) {
-        $delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff
-        Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)."
-        Start-Sleep -Seconds $delayInSeconds
-      }
-      else {
-        throw "Unable to download file in $maxRetries attempts."
-      }
-
-    }
+    Retry({
+      Write-Host "GET $uri"
+      Invoke-WebRequest $uri -OutFile $installScript
+    })
   }
 
   return $installScript
@@ -308,8 +312,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
 
   # If the version of msbuild is going to be xcopied,
   # use this version. Version matches a package here:
-  # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=16.8.0-preview3&view=overview
-  $defaultXCopyMSBuildVersion = '16.8.0-preview3'
+  # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=16.10.0-preview2&view=overview
+  $defaultXCopyMSBuildVersion = '16.10.0-preview2'
 
   if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs }
   $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { $vsMinVersionReqdStr }
@@ -403,9 +407,13 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) {
     }
 
     Create-Directory $packageDir
+
     Write-Host "Downloading $packageName $packageVersion"
     $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
-    Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -OutFile $packagePath
+    Retry({
+      Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -OutFile $packagePath
+    })
+
     Unzip $packagePath $packageDir
   }
 
@@ -442,27 +450,9 @@ function LocateVisualStudio([object]$vsRequirements = $null){
   if (!(Test-Path $vsWhereExe)) {
     Create-Directory $vsWhereDir
     Write-Host 'Downloading vswhere'
-    $maxRetries = 5
-    $retries = 1
-
-    while($true) {
-      try {
-        Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe
-        break
-      }
-      catch{
-        Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
-      }
-
-      if (++$retries -le $maxRetries) {
-        $delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff
-        Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)."
-        Start-Sleep -Seconds $delayInSeconds
-      }
-      else {
-        Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to download file in $maxRetries attempts."
-      }
-    }
+    Retry({
+      Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe
+    })
   }
 
   if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs }
@@ -498,7 +488,7 @@ function InitializeBuildTool() {
   if (Test-Path variable:global:_BuildTool) {
     # If the requested msbuild parameters do not match, clear the cached variables.
     if($global:_BuildTool.Contains('ExcludePrereleaseVS') -and $global:_BuildTool.ExcludePrereleaseVS -ne $excludePrereleaseVS) {
-      Remove-Item variable:global:_BuildTool 
+      Remove-Item variable:global:_BuildTool
       Remove-Item variable:global:_MSBuildExe
     } else {
       return $global:_BuildTool
@@ -555,7 +545,7 @@ function GetDefaultMSBuildEngine() {
 
 function GetNuGetPackageCachePath() {
   if ($env:NUGET_PACKAGES -eq $null) {
-    # Use local cache on CI to ensure deterministic build. 
+    # Use local cache on CI to ensure deterministic build.
     # Avoid using the http cache as workaround for https://github.com/NuGet/Home/issues/3116
     # use global cache in dev builds to avoid cost of downloading packages.
     # For directory normalization, see also: https://github.com/NuGet/Home/issues/7968
@@ -712,7 +702,10 @@ function MSBuild-Core() {
   }
 
   foreach ($arg in $args) {
-    if ($arg -ne $null -and $arg.Trim() -ne "") {
+    if ($null -ne $arg -and $arg.Trim() -ne "") {
+      if ($arg.EndsWith('\')) {
+        $arg = $arg + "\"
+      }
       $cmdArgs += " `"$arg`""
     }
   }
@@ -784,7 +777,7 @@ function Get-Darc($version) {
 
 . $PSScriptRoot\pipeline-logging-functions.ps1
 
-$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..')
+$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..\')
 $EngRoot = Resolve-Path (Join-Path $PSScriptRoot '..')
 $ArtifactsDir = Join-Path $RepoRoot 'artifacts'
 $ToolsetDir = Join-Path $ArtifactsDir 'toolset'

eng/common/tools.sh

@@ -485,13 +485,14 @@ _script_dir=`dirname "$_ResolvePath"`
 
 eng_root=`cd -P "$_script_dir/.." && pwd`
 repo_root=`cd -P "$_script_dir/../.." && pwd`
-artifacts_dir="$repo_root/artifacts"
+repo_root="${repo_root}/"
+artifacts_dir="${repo_root}artifacts"
 toolset_dir="$artifacts_dir/toolset"
-tools_dir="$repo_root/.tools"
+tools_dir="${repo_root}.tools"
 log_dir="$artifacts_dir/log/$configuration"
 temp_dir="$artifacts_dir/tmp/$configuration"
 
-global_json_file="$repo_root/global.json"
+global_json_file="${repo_root}global.json"
 # determine if global.json contains a "runtimes" entry
 global_json_has_runtimes=false
 if command -v jq &> /dev/null; then
@@ -504,7 +505,7 @@ fi
 
 # HOME may not be defined in some scenarios, but it is required by NuGet
 if [[ -z $HOME ]]; then
-  export HOME="$repo_root/artifacts/.home/"
+  export HOME="${repo_root}artifacts/.home/"
   mkdir -p "$HOME"
 fi
 

global.json

@@ -13,7 +13,7 @@
     }
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21303.2",
+    "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21319.2",
     "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19069.2"
   }
 }