[master] Update dependencies from dotnet/arcade (#532)

* Update dependencies from https://github.com/dotnet/arcade build 20190626.44

- Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19326.44
- Microsoft.DotNet.GenAPI - 1.0.0-beta.19326.44
- Microsoft.DotNet.Helix.Sdk - 2.0.0-beta.19326.44

* Update dependencies from https://github.com/dotnet/arcade build 20190627.39

- Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19327.39
- Microsoft.DotNet.GenAPI - 1.0.0-beta.19327.39
- Microsoft.DotNet.Helix.Sdk - 2.0.0-beta.19327.39

* Update dependencies from https://github.com/dotnet/arcade build 20190628.2

- Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19328.2
- Microsoft.DotNet.GenAPI - 1.0.0-beta.19328.2
- Microsoft.DotNet.Helix.Sdk - 2.0.0-beta.19328.2

* Update dependencies from https://github.com/dotnet/arcade build 20190629.2

- Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19329.2
- Microsoft.DotNet.GenAPI - 1.0.0-beta.19329.2
- Microsoft.DotNet.Helix.Sdk - 2.0.0-beta.19329.2

* Update dependencies from https://github.com/dotnet/arcade build 20190630.1

- Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19330.1
- Microsoft.DotNet.GenAPI - 1.0.0-beta.19330.1
- Microsoft.DotNet.Helix.Sdk - 2.0.0-beta.19330.1

* Add restore sources to Versions.props

eng/Version.Details.xml

@@ -11,17 +11,17 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19326.2">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19330.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>4b3d46cc75969c4e2de5786ec2b10a430b26dd9f</Sha>
+      <Sha>89fab80685c91024c8f9e21f1c37f62580f648f8</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="2.0.0-beta.19326.2">
+    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="2.0.0-beta.19330.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>4b3d46cc75969c4e2de5786ec2b10a430b26dd9f</Sha>
+      <Sha>89fab80685c91024c8f9e21f1c37f62580f648f8</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.GenAPI" Version="1.0.0-beta.19326.2">
+    <Dependency Name="Microsoft.DotNet.GenAPI" Version="1.0.0-beta.19330.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>4b3d46cc75969c4e2de5786ec2b10a430b26dd9f</Sha>
+      <Sha>89fab80685c91024c8f9e21f1c37f62580f648f8</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/Versions.props

@@ -6,6 +6,17 @@
     <SystemDeviceGpioPackageVersion>0.1.0-prerelease.19327.1</SystemDeviceGpioPackageVersion>
     <IotDeviceBindingsPackageVersion>0.1.0-prerelease.19327.1</IotDeviceBindingsPackageVersion>
     <!-- dotnet/arcade dependencies -->
-    <MicrosoftDotNetGenAPIPackageVersion>1.0.0-beta.19326.2</MicrosoftDotNetGenAPIPackageVersion>
+    <MicrosoftDotNetGenAPIPackageVersion>1.0.0-beta.19330.1</MicrosoftDotNetGenAPIPackageVersion>
   </PropertyGroup>
-</Project>
+  <!-- Restore sources -->
+  <PropertyGroup>
+    <RestoreSources>
+      $(RestoreSources);
+      https://dotnetfeed.blob.core.windows.net/dotnet-tools-internal/index.json;
+      https://dotnetfeed.blob.core.windows.net/dotnet-iot/index.json;
+      https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json;
+      https://dotnet.myget.org/F/dotnet-core/api/v3/index.json;
+      https://api.nuget.org/v3/index.json;
+    </RestoreSources>
+  </PropertyGroup>
+</Project>
\ No newline at end of file

eng/common/post-build/nuget-validation.ps1

+# This script validates NuGet package metadata information using this 
+# tool: https://github.com/NuGet/NuGetGallery/tree/jver-verify/src/VerifyMicrosoftPackage
+
+param(
+  [Parameter(Mandatory=$true)][string] $PackagesPath,           # Path to where the packages to be validated are
+  [Parameter(Mandatory=$true)][string] $ToolDestinationPath     # Where the validation tool should be downloaded to
+)
+
+$ErrorActionPreference = "Stop"
+Set-StrictMode -Version 2.0
+
+. $PSScriptRoot\..\tools.ps1
+
+try {
+  $url = "https://raw.githubusercontent.com/NuGet/NuGetGallery/jver-verify/src/VerifyMicrosoftPackage/verify.ps1" 
+
+  New-Item -ItemType "directory" -Path ${ToolDestinationPath} -Force
+
+  Invoke-WebRequest $url -OutFile ${ToolDestinationPath}\verify.ps1 
+
+  & ${ToolDestinationPath}\verify.ps1 ${PackagesPath}\*.nupkg
+} 
+catch {
+  Write-PipelineTaskError "NuGet package validation failed. Please check error logs."
+  Write-Host $_
+  Write-Host $_.ScriptStackTrace
+  ExitWithExitCode 1
+}

eng/common/post-build/promote-build.ps1

+param(
+  [Parameter(Mandatory=$true)][int] $BuildId,
+  [Parameter(Mandatory=$true)][int] $ChannelId,
+  [Parameter(Mandatory=$true)][string] $BarToken,
+  [string] $MaestroEndpoint = "https://maestro-prod.westus2.cloudapp.azure.com",
+  [string] $ApiVersion = "2019-01-16"
+)
+
+$ErrorActionPreference = "Stop"
+Set-StrictMode -Version 2.0
+
+. $PSScriptRoot\..\tools.ps1
+
+function Get-Headers([string]$accept, [string]$barToken) {
+  $headers = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]'
+  $headers.Add('Accept',$accept)
+  $headers.Add('Authorization',"Bearer $barToken")
+  return $headers
+}
+
+try {
+  $maestroHeaders = Get-Headers 'application/json' $BarToken
+
+  # Get info about which channels the build has already been promoted to
+  $getBuildApiEndpoint = "$MaestroEndpoint/api/builds/${BuildId}?api-version=$ApiVersion"
+  $buildInfo = Invoke-WebRequest -Method Get -Uri $getBuildApiEndpoint -Headers $maestroHeaders | ConvertFrom-Json
+
+  if (!$buildInfo) {
+    Write-Host "Build with BAR ID $BuildId was not found in BAR!"
+    ExitWithExitCode 1
+  }
+
+  # Find whether the build is already assigned to the channel or not
+  if ($buildInfo.channels) {
+    foreach ($channel in $buildInfo.channels) {
+      if ($channel.Id -eq $ChannelId) {
+        Write-Host "The build with BAR ID $BuildId is already on channel $ChannelId!"
+        ExitWithExitCode 0
+      }
+    }
+  }
+
+  Write-Host "Build not present in channel $ChannelId. Promoting build ... "
+
+  $promoteBuildApiEndpoint = "$maestroEndpoint/api/channels/${ChannelId}/builds/${BuildId}?api-version=$ApiVersion"
+  Invoke-WebRequest -Method Post -Uri $promoteBuildApiEndpoint -Headers $maestroHeaders
+  Write-Host "done."
+} 
+catch {
+  Write-Host "There was an error while trying to promote build '$BuildId' to channel '$ChannelId'"
+  Write-Host $_
+  Write-Host $_.ScriptStackTrace
+}

eng/common/post-build/trigger-subscriptions.ps1

+param(
+  [Parameter(Mandatory=$true)][string] $SourceRepo,
+  [Parameter(Mandatory=$true)][int] $ChannelId,
+  [string] $MaestroEndpoint = "https://maestro-prod.westus2.cloudapp.azure.com",
+  [string] $BarToken,
+  [string] $ApiVersion = "2019-01-16"
+)
+
+$ErrorActionPreference = "Stop"
+Set-StrictMode -Version 2.0
+
+. $PSScriptRoot\..\tools.ps1
+
+function Get-Headers([string]$accept, [string]$barToken) {
+  $headers = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]'
+  $headers.Add('Accept',$accept)
+  $headers.Add('Authorization',"Bearer $barToken")
+  return $headers
+}
+
+# Get all the $SourceRepo subscriptions
+$normalizedSurceRepo = $SourceRepo.Replace('dnceng@', '')
+$getSubscriptionsApiEndpoint = "$maestroEndpoint/api/subscriptions?sourceRepository=$normalizedSurceRepo&api-version=$apiVersion"
+$headers = Get-Headers 'application/json' $barToken
+
+$subscriptions = Invoke-WebRequest -Uri $getSubscriptionsApiEndpoint -Headers $headers | ConvertFrom-Json
+
+if (!$subscriptions) {
+  Write-Host "No subscriptions found for source repo '$normalizedSurceRepo' in channel '$ChannelId'"
+  return
+}
+
+$subscriptionsToTrigger = New-Object System.Collections.Generic.List[string]
+$failedTriggeredSubscription = $false
+
+# Get all enabled subscriptions that need dependency flow on 'everyBuild'
+foreach ($subscription in $subscriptions) {
+  if ($subscription.enabled -and $subscription.policy.updateFrequency -like 'everyBuild' -and $subscription.channel.id -eq $ChannelId) {
+    Write-Host "$subscription.id"
+    [void]$subscriptionsToTrigger.Add($subscription.id)
+  }
+}
+
+foreach ($subscriptionToTrigger in $subscriptionsToTrigger) {
+  try {
+    $triggerSubscriptionApiEndpoint = "$maestroEndpoint/api/subscriptions/$subscriptionToTrigger/trigger?api-version=$apiVersion"
+    $headers = Get-Headers 'application/json' $BarToken
+    
+    Write-Host "Triggering subscription '$subscriptionToTrigger'..."
+
+    Invoke-WebRequest -Uri $triggerSubscriptionApiEndpoint -Headers $headers -Method Post
+  
+    Write-Host "Subscription '$subscriptionToTrigger' triggered!"
+  } 
+  catch
+  {
+    Write-Host "There was an error while triggering subscription '$subscriptionToTrigger'"
+    Write-Host $_
+    Write-Host $_.ScriptStackTrace
+    $failedTriggeredSubscription = $true
+  }
+}
+
+if ($failedTriggeredSubscription) {
+  Write-Host "At least one subscription failed to be triggered..."
+  ExitWithExitCode 1
+}
+
+Write-Host "All subscriptions were triggered successfully!"

eng/common/sdl/extract-artifact-packages.ps1

+param(
+  [Parameter(Mandatory=$true)][string] $InputPath,              # Full path to directory where artifact packages are stored
+  [Parameter(Mandatory=$true)][string] $ExtractPath            # Full path to directory where the packages will be extracted
+)
+
+$ErrorActionPreference = "Stop"
+Set-StrictMode -Version 2.0
+$ExtractPackage = {
+  param( 
+    [string] $PackagePath                                 # Full path to a NuGet package
+  )
+  
+  if (!(Test-Path $PackagePath)) {
+    Write-PipelineTaskError "Input file does not exist: $PackagePath"
+    ExitWithExitCode 1
+  }
+  
+  $RelevantExtensions = @(".dll", ".exe", ".pdb")
+  Write-Host -NoNewLine "Extracting" ([System.IO.Path]::GetFileName($PackagePath)) "... "
+
+  $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
+  $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId
+
+  Add-Type -AssemblyName System.IO.Compression.FileSystem
+
+  [System.IO.Directory]::CreateDirectory($ExtractPath);
+
+  try {
+    $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath)
+
+    $zip.Entries | 
+    Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} |
+      ForEach-Object {
+          $TargetFile = Join-Path -Path $ExtractPath -ChildPath $_.Name
+
+          [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true)
+        }
+  }
+  catch {
+  
+  }
+  finally {
+    $zip.Dispose() 
+  }
+ }
+ function ExtractArtifacts {
+  if (!(Test-Path $InputPath)) {
+    Write-Host "Input Path does not exist: $InputPath"
+    ExitWithExitCode 0
+  }
+  $Jobs = @()
+  Get-ChildItem "$InputPath\*.nupkg" |
+    ForEach-Object {
+      $Jobs += Start-Job -ScriptBlock $ExtractPackage -ArgumentList $_.FullName
+    }
+
+  foreach ($Job in $Jobs) {
+    Wait-Job -Id $Job.Id | Receive-Job
+  }
+}
+
+try {
+  Measure-Command { ExtractArtifacts }
+}
+catch {
+  Write-Host $_
+  Write-Host $_.Exception
+  Write-Host $_.ScriptStackTrace
+  ExitWithExitCode 1
+}

eng/common/templates/job/execute-sdl.yml

@@ -20,6 +20,16 @@ jobs:
       downloadType: specific files
       matchingPattern: "**"
       downloadPath: $(Build.SourcesDirectory)\artifacts
+  - powershell: eng/common/sdl/extract-artifact-packages.ps1
+      -InputPath $(Build.SourcesDirectory)\artifacts\BlobArtifacts
+      -ExtractPath $(Build.SourcesDirectory)\artifacts\BlobArtifacts
+    displayName: Extract Blob Artifacts
+    continueOnError: ${{ parameters.continueOnError }}
+  - powershell: eng/common/sdl/extract-artifact-packages.ps1
+      -InputPath $(Build.SourcesDirectory)\artifacts\PackageArtifacts
+      -ExtractPath $(Build.SourcesDirectory)\artifacts\PackageArtifacts
+    displayName: Extract Package Artifacts
+    continueOnError: ${{ parameters.continueOnError }}
   - task: NuGetToolInstaller@1
     displayName: 'Install NuGet.exe'
   - task: NuGetCommand@2

eng/common/templates/post-build/channels/internal-servicing.yml

@@ -36,7 +36,7 @@ stages:
             /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
             /p:Configuration=Release
 
-  - job:
+  - job: publish_assets
     displayName: Publish Assets
     dependsOn: setupMaestroVars
     variables:
@@ -111,7 +111,10 @@ stages:
             -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw)
         enabled: false
 
-
+      - template: ../trigger-subscription.yml
+        parameters:
+          ChannelId: ${{ variables.InternalServicing_30_Channel_Id }}
+        
 - stage: IS_PublishValidation
   displayName: Publish Validation
   variables:
@@ -164,4 +167,4 @@ stages:
 
   - template: ../promote-build.yml
     parameters:
-      ChannelId: ${{ variables.InternalServicing_30_Channel_Id }}
+      ChannelId: ${{ variables.InternalServicing_30_Channel_Id }}
\ No newline at end of file

eng/common/templates/post-build/channels/public-dev-release.yml

@@ -160,3 +160,7 @@ stages:
           targetType: inline
           script: |
             darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com/ --password $(MaestroAccessToken) --latest-location
+
+  - template: ../promote-build.yml	
+    parameters:	
+      ChannelId: ${{ variables.PublicDevRelease_30_Channel_Id }}

eng/common/templates/post-build/channels/public-release.yml

@@ -36,7 +36,7 @@ stages:
             /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
             /p:Configuration=Release
 
-  - job:
+  - job: publish_assets
     displayName: Publish Assets
     dependsOn: setupMaestroVars
     variables:
@@ -111,6 +111,9 @@ stages:
             -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw)
         enabled: false
 
+      - template: ../trigger-subscription.yml
+        parameters:
+          ChannelId: ${{ variables.PublicRelease_30_Channel_Id }}
 
 - stage: PubRel_PublishValidation
   displayName: Publish Validation

eng/common/templates/post-build/channels/public-validation-release.yml

@@ -113,3 +113,7 @@ stages:
           targetType: inline
           script: |
             darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com --password $(MaestroAccessToken) --latest-location
+
+  - template: ../promote-build.yml	
+    parameters:	
+      ChannelId: ${{ variables.PublicValidationRelease_30_Channel_Id }}

eng/common/templates/post-build/post-build.yml

@@ -2,6 +2,7 @@ parameters:
   enableSourceLinkValidation: true
   enableSigningValidation: true
   enableSymbolValidation: true
+  enableNugetValidation: true
   SDLValidationParameters:
     enable: false
     params: ''
@@ -11,6 +12,25 @@ stages:
   dependsOn: build
   displayName: Validate
   jobs:
+  - ${{ if eq(parameters.enableNugetValidation, 'true') }}:
+    - job:
+      displayName: NuGet Validation
+      pool:
+        vmImage: 'windows-2019'
+      steps:
+        - task: DownloadBuildArtifacts@0
+          displayName: Download Package Artifacts
+          inputs:
+            buildType: current
+            artifactName: PackageArtifacts
+
+        - task: PowerShell@2
+          displayName: Validate
+          inputs:
+            filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1
+            arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ 
+              -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ 
+
   - ${{ if eq(parameters.enableSigningValidation, 'true') }}:
     - job:
       displayName: Signing Validation

eng/common/templates/post-build/promote-build.yml

@@ -18,10 +18,7 @@ jobs:
     - task: PowerShell@2
       displayName: Add Build to Channel
       inputs:
-        targetType: inline
-        script: |
-          $headers = @{
-            "Accept" = "application/json"
-            "Authorization" = "Bearer $(MaestroAccessToken)"
-          }
-          Invoke-RestMethod -Method Post -Headers $headers -Uri https://maestro-prod.westus2.cloudapp.azure.com/api/channels/$(ChannelId)/builds/$(BARBuildId)?api-version=2019-01-16
+        filePath: $(Build.SourcesDirectory)/eng/common/post-build/promote-build.ps1
+        arguments: -BuildId $(BARBuildId) 
+          -ChannelId $(ChannelId) 
+          -BarToken $(MaestroAccessToken)

eng/common/templates/post-build/trigger-subscription.yml

+parameters:
+  ChannelId: 0
+
+steps:
+- task: PowerShell@2
+  displayName: Triggering subscriptions
+  inputs:
+    filePath: $(Build.SourcesDirectory)/eng/common/post-build/trigger-subscriptions.ps1
+    arguments: -SourceRepo $(Build.Repository.Uri)
+      -ChannelId ${{ parameters.ChannelId }}
+      -BarToken $(MaestroAccessTokenInt)
\ No newline at end of file

global.json

 {
   "tools": {
-    "dotnet": "3.0.100-preview5-011568",
+    "dotnet": "3.0.100-preview6-012264",
     "runtimes": {
       "dotnet/x64": [
         "2.1.11"
@@ -8,10 +8,10 @@
     }
   },
   "sdk": {
-    "version": "3.0.100-preview5-011568"
+    "version": "3.0.100-preview6-012264"
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19326.2",
-    "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19326.2"
+    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19330.1",
+    "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19330.1"
   }
 }