[master] Update dependencies from dotnet/arcade (#37001)

* Update dependencies from https://github.com/dotnet/arcade build 20190704.2

- Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19354.2

* Update dependencies from https://github.com/dotnet/arcade build 20190705.2

- Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19355.2

* Update dependencies from https://github.com/dotnet/arcade build 20190706.1

- Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19356.1

* Update dependencies from https://github.com/dotnet/arcade build 20190708.1

- Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19358.1

* Update dependencies from https://github.com/dotnet/arcade build 20190709.1

- Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19359.1

eng/Version.Details.xml

@@ -3,9 +3,9 @@
   <ProductDependencies>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19326.44">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19359.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>d39a62deaf3aa4e03c0b7dadc320a517e0e00187</Sha>
+      <Sha>ef3834feb8615429a58808cdcf9ad9284d767654</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/Versions.props

@@ -9,7 +9,6 @@
     <MinorVersion>3</MinorVersion>
     <PatchVersion>0</PatchVersion>
     <PreReleaseVersionLabel>beta2</PreReleaseVersionLabel>
-
     <VersionPrefix>$(MajorVersion).$(MinorVersion).$(PatchVersion)</VersionPrefix>
     <SemanticVersioningV1>true</SemanticVersioningV1>
     <!-- 

eng/common/post-build/nuget-validation.ps1

+# This script validates NuGet package metadata information using this 
+# tool: https://github.com/NuGet/NuGetGallery/tree/jver-verify/src/VerifyMicrosoftPackage
+
+param(
+  [Parameter(Mandatory=$true)][string] $PackagesPath,           # Path to where the packages to be validated are
+  [Parameter(Mandatory=$true)][string] $ToolDestinationPath     # Where the validation tool should be downloaded to
+)
+
+$ErrorActionPreference = "Stop"
+Set-StrictMode -Version 2.0
+
+. $PSScriptRoot\..\tools.ps1
+
+try {
+  $url = "https://raw.githubusercontent.com/NuGet/NuGetGallery/jver-verify/src/VerifyMicrosoftPackage/verify.ps1" 
+
+  New-Item -ItemType "directory" -Path ${ToolDestinationPath} -Force
+
+  Invoke-WebRequest $url -OutFile ${ToolDestinationPath}\verify.ps1 
+
+  & ${ToolDestinationPath}\verify.ps1 ${PackagesPath}\*.nupkg
+} 
+catch {
+  Write-PipelineTaskError "NuGet package validation failed. Please check error logs."
+  Write-Host $_
+  Write-Host $_.ScriptStackTrace
+  ExitWithExitCode 1
+}

eng/common/post-build/trigger-subscriptions.ps1

+param(
+  [Parameter(Mandatory=$true)][string] $SourceRepo,
+  [Parameter(Mandatory=$true)][int] $ChannelId,
+  [string] $MaestroEndpoint = "https://maestro-prod.westus2.cloudapp.azure.com",
+  [string] $BarToken,
+  [string] $ApiVersion = "2019-01-16"
+)
+
+$ErrorActionPreference = "Stop"
+Set-StrictMode -Version 2.0
+
+. $PSScriptRoot\..\tools.ps1
+
+function Get-Headers([string]$accept, [string]$barToken) {
+  $headers = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]'
+  $headers.Add('Accept',$accept)
+  $headers.Add('Authorization',"Bearer $barToken")
+  return $headers
+}
+
+# Get all the $SourceRepo subscriptions
+$normalizedSurceRepo = $SourceRepo.Replace('dnceng@', '')
+$getSubscriptionsApiEndpoint = "$maestroEndpoint/api/subscriptions?sourceRepository=$normalizedSurceRepo&api-version=$apiVersion"
+$headers = Get-Headers 'application/json' $barToken
+
+$subscriptions = Invoke-WebRequest -Uri $getSubscriptionsApiEndpoint -Headers $headers | ConvertFrom-Json
+
+if (!$subscriptions) {
+  Write-Host "No subscriptions found for source repo '$normalizedSurceRepo' in channel '$ChannelId'"
+  return
+}
+
+$subscriptionsToTrigger = New-Object System.Collections.Generic.List[string]
+$failedTriggeredSubscription = $false
+
+# Get all enabled subscriptions that need dependency flow on 'everyBuild'
+foreach ($subscription in $subscriptions) {
+  if ($subscription.enabled -and $subscription.policy.updateFrequency -like 'everyBuild' -and $subscription.channel.id -eq $ChannelId) {
+    Write-Host "$subscription.id"
+    [void]$subscriptionsToTrigger.Add($subscription.id)
+  }
+}
+
+foreach ($subscriptionToTrigger in $subscriptionsToTrigger) {
+  try {
+    $triggerSubscriptionApiEndpoint = "$maestroEndpoint/api/subscriptions/$subscriptionToTrigger/trigger?api-version=$apiVersion"
+    $headers = Get-Headers 'application/json' $BarToken
+    
+    Write-Host "Triggering subscription '$subscriptionToTrigger'..."
+
+    Invoke-WebRequest -Uri $triggerSubscriptionApiEndpoint -Headers $headers -Method Post
+  
+    Write-Host "Subscription '$subscriptionToTrigger' triggered!"
+  } 
+  catch
+  {
+    Write-Host "There was an error while triggering subscription '$subscriptionToTrigger'"
+    Write-Host $_
+    Write-Host $_.ScriptStackTrace
+    $failedTriggeredSubscription = $true
+  }
+}
+
+if ($failedTriggeredSubscription) {
+  Write-Host "At least one subscription failed to be triggered..."
+  ExitWithExitCode 1
+}
+
+Write-Host "All subscriptions were triggered successfully!"

eng/common/sdl/execute-all-sdl-tools.ps1

@@ -22,7 +22,9 @@ Param(
   [string] $TsaCodebaseAdmin,                                                              # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs.
   [string] $TsaBugAreaPath,                                                                # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs.
   [string] $TsaIterationPath,                                                              # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs.
-  [string] $GuardianLoggerLevel="Standard"                                                 # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error
+  [string] $GuardianLoggerLevel="Standard",                                                # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error
+  [string[]] $CrScanAdditionalRunConfigParams,                                             # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1")
+  [string[]] $PoliCheckAdditionalRunConfigParams                                           # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1")
 )
 
 $ErrorActionPreference = "Stop"
@@ -69,10 +71,10 @@ if ($TsaOnboard) {
 }
 
 if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) {
-  & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel
+  & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
 }
 if ($SourceToolsList -and $SourceToolsList.Count -gt 0) {
-  & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel
+  & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
 }
 
 if ($UpdateBaseline) {

eng/common/sdl/extract-artifact-packages.ps1

@@ -44,6 +44,10 @@ $ExtractPackage = {
   }
  }
  function ExtractArtifacts {
+  if (!(Test-Path $InputPath)) {
+    Write-Host "Input Path does not exist: $InputPath"
+    ExitWithExitCode 0
+  }
   $Jobs = @()
   Get-ChildItem "$InputPath\*.nupkg" |
     ForEach-Object {

eng/common/sdl/run-sdl.ps1

@@ -5,7 +5,9 @@ Param(
   [string] $GdnFolder,
   [string[]] $ToolsList,
   [string] $UpdateBaseline,
-  [string] $GuardianLoggerLevel="Standard"
+  [string] $GuardianLoggerLevel="Standard",
+  [string[]] $CrScanAdditionalRunConfigParams,
+  [string[]] $PoliCheckAdditionalRunConfigParams
 )
 
 $ErrorActionPreference = "Stop"
@@ -29,8 +31,8 @@ foreach ($tool in $ToolsList) {
   Write-Host $tool
   # We have to manually configure tools that run on source to look at the source directory only
   if ($tool -eq "credscan") {
-    Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory : $TargetDirectory `""
-    & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory : $TargetDirectory "
+    Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory : $TargetDirectory `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})"
+    & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory : $TargetDirectory " $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})
     if ($LASTEXITCODE -ne 0) {
       Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE."
       exit $LASTEXITCODE
@@ -38,8 +40,8 @@ foreach ($tool in $ToolsList) {
     $config = $True
   }
   if ($tool -eq "policheck") {
-    Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target : $TargetDirectory `""
-    & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target : $TargetDirectory "
+    Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target : $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})"
+    & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target : $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})
     if ($LASTEXITCODE -ne 0) {
       Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE."
       exit $LASTEXITCODE

eng/common/templates/post-build/channels/internal-servicing.yml

@@ -36,7 +36,7 @@ stages:
             /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
             /p:Configuration=Release
 
-  - job:
+  - job: publish_assets
     displayName: Publish Assets
     dependsOn: setupMaestroVars
     variables:
@@ -111,7 +111,10 @@ stages:
             -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw)
         enabled: false
 
-
+      - template: ../trigger-subscription.yml
+        parameters:
+          ChannelId: ${{ variables.InternalServicing_30_Channel_Id }}
+        
 - stage: IS_PublishValidation
   displayName: Publish Validation
   variables:
@@ -164,4 +167,4 @@ stages:
 
   - template: ../promote-build.yml
     parameters:
-      ChannelId: ${{ variables.InternalServicing_30_Channel_Id }}
+      ChannelId: ${{ variables.InternalServicing_30_Channel_Id }}
\ No newline at end of file

eng/common/templates/post-build/channels/public-release.yml

@@ -36,7 +36,7 @@ stages:
             /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
             /p:Configuration=Release
 
-  - job:
+  - job: publish_assets
     displayName: Publish Assets
     dependsOn: setupMaestroVars
     variables:
@@ -111,6 +111,9 @@ stages:
             -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw)
         enabled: false
 
+      - template: ../trigger-subscription.yml
+        parameters:
+          ChannelId: ${{ variables.PublicRelease_30_Channel_Id }}
 
 - stage: PubRel_PublishValidation
   displayName: Publish Validation

eng/common/templates/post-build/post-build.yml

@@ -2,6 +2,7 @@ parameters:
   enableSourceLinkValidation: true
   enableSigningValidation: true
   enableSymbolValidation: true
+  enableNugetValidation: true
   SDLValidationParameters:
     enable: false
     params: ''
@@ -11,6 +12,25 @@ stages:
   dependsOn: build
   displayName: Validate
   jobs:
+  - ${{ if eq(parameters.enableNugetValidation, 'true') }}:
+    - job:
+      displayName: NuGet Validation
+      pool:
+        vmImage: 'windows-2019'
+      steps:
+        - task: DownloadBuildArtifacts@0
+          displayName: Download Package Artifacts
+          inputs:
+            buildType: current
+            artifactName: PackageArtifacts
+
+        - task: PowerShell@2
+          displayName: Validate
+          inputs:
+            filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1
+            arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ 
+              -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ 
+
   - ${{ if eq(parameters.enableSigningValidation, 'true') }}:
     - job:
       displayName: Signing Validation

eng/common/templates/post-build/trigger-subscription.yml

+parameters:
+  ChannelId: 0
+
+steps:
+- task: PowerShell@2
+  displayName: Triggering subscriptions
+  inputs:
+    filePath: $(Build.SourcesDirectory)/eng/common/post-build/trigger-subscriptions.ps1
+    arguments: -SourceRepo $(Build.Repository.Uri)
+      -ChannelId ${{ parameters.ChannelId }}
+      -BarToken $(MaestroAccessTokenInt)
\ No newline at end of file

global.json

@@ -7,6 +7,6 @@
     "xcopy-msbuild": "16.0.0-alpha"
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19326.44"
+    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19359.1"
   }
 }