[main] Update dependencies from dotnet/arcade (#1582)

* Update dependencies from https://github.com/dotnet/arcade build 20210701.2

Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.GenAPI , Microsoft.DotNet.Helix.Sdk
 From Version 6.0.0-beta.21330.2 -> To Version 6.0.0-beta.21351.2

* Update dependencies from https://github.com/dotnet/arcade build 20210705.2

Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.GenAPI , Microsoft.DotNet.Helix.Sdk
 From Version 6.0.0-beta.21330.2 -> To Version 6.0.0-beta.21355.2

* Update dependencies from https://github.com/dotnet/arcade build 20210707.3

Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.GenAPI , Microsoft.DotNet.Helix.Sdk
 From Version 6.0.0-beta.21330.2 -> To Version 6.0.0-beta.21357.3

* Update dependencies from https://github.com/dotnet/arcade build 20210709.3

Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.GenAPI , Microsoft.DotNet.Helix.Sdk
 From Version 6.0.0-beta.21330.2 -> To Version 6.0.0-beta.21359.3

* Update dependencies from https://github.com/dotnet/arcade build 20210713.2

Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.GenAPI , Microsoft.DotNet.Helix.Sdk
 From Version 6.0.0-beta.21330.2 -> To Version 6.0.0-beta.21363.2

* Update dependencies from https://github.com/dotnet/arcade build 20210714.3

Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.GenAPI , Microsoft.DotNet.Helix.Sdk
 From Version 6.0.0-beta.21330.2 -> To Version 6.0.0-beta.21364.3
Co-authored-by: Ndotnet-maestro[bot] <dotnet-maestro[bot]@users.noreply.github.com>

eng/Version.Details.xml

 <?xml version="1.0" encoding="utf-8"?>
 <Dependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21330.2">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21364.3">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>26345756f99087811b1fe4d02ff213eb172ec506</Sha>
+      <Sha>6a8491b61e0c243cbb6a7ff4966b48e6d1e075b1</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="6.0.0-beta.21330.2">
+    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="6.0.0-beta.21364.3">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>26345756f99087811b1fe4d02ff213eb172ec506</Sha>
+      <Sha>6a8491b61e0c243cbb6a7ff4966b48e6d1e075b1</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.GenAPI" Version="6.0.0-beta.21330.2">
+    <Dependency Name="Microsoft.DotNet.GenAPI" Version="6.0.0-beta.21364.3">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>26345756f99087811b1fe4d02ff213eb172ec506</Sha>
+      <Sha>6a8491b61e0c243cbb6a7ff4966b48e6d1e075b1</Sha>
     </Dependency>
   </ToolsetDependencies>
   <!-- ProductDependencies -->

eng/Versions.props

@@ -2,7 +2,7 @@
 <Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <PropertyGroup>
     <PreReleaseVersionLabel>prerelease</PreReleaseVersionLabel>
-    <MicrosoftDotNetGenAPIPackageVersion>6.0.0-beta.21330.2</MicrosoftDotNetGenAPIPackageVersion>
+    <MicrosoftDotNetGenAPIPackageVersion>6.0.0-beta.21364.3</MicrosoftDotNetGenAPIPackageVersion>
     <!-- dotnet/corefx dependencies -->
     <SystemDrawingCommonPackageVersion>5.0.2</SystemDrawingCommonPackageVersion>
     <SystemIOPortsPackageVersion>5.0.1</SystemIOPortsPackageVersion>

eng/common/SetupNugetSources.ps1

@@ -158,4 +158,10 @@ if ($dotnet5Source -ne $null) {
     AddPackageSource -Sources $sources -SourceName "dotnet5-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
 }
 
+$dotnet6Source = $sources.SelectSingleNode("add[@key='dotnet6']")
+if ($dotnet6Source -ne $null) {
+    AddPackageSource -Sources $sources -SourceName "dotnet6-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
+    AddPackageSource -Sources $sources -SourceName "dotnet6-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
+}
+
 $doc.Save($filename)

eng/common/SetupNugetSources.sh

@@ -129,6 +129,30 @@ if [ "$?" == "0" ]; then
     PackageSources+=('dotnet5-internal-transport')
 fi
 
+# Ensure dotnet6-internal and dotnet6-internal-transport are in the packageSources if the public dotnet6 feeds are present
+grep -i "<add key=\"dotnet6\"" $ConfigFile
+if [ "$?" == "0" ]; then
+    grep -i "<add key=\"dotnet6-internal\"" $ConfigFile
+    if [ "$?" != "0" ]; then
+        echo "Adding dotnet6-internal to the packageSources."
+        PackageSourcesNodeFooter="</packageSources>"
+        PackageSourceTemplate="${TB}<add key=\"dotnet6-internal\" value=\"https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal/nuget/v2\" />"
+
+        sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
+    fi
+    PackageSources+=('dotnet6-internal')
+
+    grep -i "<add key=\"dotnet6-internal-transport\">" $ConfigFile
+    if [ "$?" != "0" ]; then
+        echo "Adding dotnet6-internal-transport to the packageSources."
+        PackageSourcesNodeFooter="</packageSources>"
+        PackageSourceTemplate="${TB}<add key=\"dotnet6-internal-transport\" value=\"https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal-transport/nuget/v2\" />"
+
+        sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
+    fi
+    PackageSources+=('dotnet6-internal-transport')
+fi
+
 # I want things split line by line
 PrevIFS=$IFS
 IFS=$'\n'

eng/common/cross/build-rootfs.sh

@@ -33,7 +33,6 @@ __AlpinePackages="alpine-base"
 __AlpinePackages+=" build-base"
 __AlpinePackages+=" linux-headers"
 __AlpinePackagesEdgeCommunity=" lldb-dev"
-__AlpinePackagesEdgeMain=" llvm10-libs"
 __AlpinePackagesEdgeMain+=" python3"
 __AlpinePackagesEdgeMain+=" libedit"
 
@@ -115,6 +114,8 @@ while :; do
             __UbuntuArch=s390x
             __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/"
             __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//')
+            __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp-dev//')
+            __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp5//')
             unset __LLDB_Package
             ;;
         x86)
@@ -191,6 +192,8 @@ while :; do
             __CodeName=alpine
             __UbuntuRepo=
             __AlpineVersion=3.9
+            __AlpinePackagesEdgeMain+=" llvm11-libs"
+            __AlpinePackagesEdgeMain+=" clang-libs"
             ;;
         alpine3.13)
             __CodeName=alpine
@@ -201,6 +204,7 @@ while :; do
             __AlpinePackagesEdgeCommunity=
             __AlpinePackages+=$__AlpinePackagesEdgeMain
             __AlpinePackagesEdgeMain=
+            __AlpinePackages+=" llvm10-libs"
             ;;
         freebsd11)
             __FreeBSDBase="11.3-RELEASE"

eng/common/sdl/configure-sdl-tool.ps1

+Param(
+  [string] $GuardianCliLocation,
+  [string] $WorkingDirectory,
+  [string] $TargetDirectory,
+  [string] $GdnFolder,
+  # The list of Guardian tools to configure. For each object in the array:
+  # - If the item is a [hashtable], it must contain these entries:
+  #   - Name = The tool name as Guardian knows it.
+  #   - Scenario = (Optional) Scenario-specific name for this configuration entry. It must be unique
+  #     among all tool entries with the same Name.
+  #   - Args = (Optional) Array of Guardian tool configuration args, like '@("Target > C:\temp")'
+  # - If the item is a [string] $v, it is treated as '@{ Name="$v" }'
+  [object[]] $ToolsList,
+  [string] $GuardianLoggerLevel='Standard',
+  # Optional: Additional params to add to any tool using CredScan.
+  [string[]] $CrScanAdditionalRunConfigParams,
+  # Optional: Additional params to add to any tool using PoliCheck.
+  [string[]] $PoliCheckAdditionalRunConfigParams
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+$disableConfigureToolsetImport = $true
+$global:LASTEXITCODE = 0
+
+try {
+  # `tools.ps1` checks $ci to perform some actions. Since the SDL
+  # scripts don't necessarily execute in the same agent that run the
+  # build.ps1/sh script this variable isn't automatically set.
+  $ci = $true
+  . $PSScriptRoot\..\tools.ps1
+
+  # Normalize tools list: all in [hashtable] form with defined values for each key.
+  $ToolsList = $ToolsList |
+    ForEach-Object {
+      if ($_ -is [string]) {
+        $_ = @{ Name = $_ }
+      }
+
+      if (-not ($_['Scenario'])) { $_.Scenario = "" }
+      if (-not ($_['Args'])) { $_.Args = @() }
+      $_
+    }
+  
+  Write-Host "List of tools to configure:"
+  $ToolsList | ForEach-Object { $_ | Out-String | Write-Host }
+
+  # We store config files in the r directory of .gdn
+  $gdnConfigPath = Join-Path $GdnFolder 'r'
+  $ValidPath = Test-Path $GuardianCliLocation
+
+  if ($ValidPath -eq $False)
+  {
+    Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location."
+    ExitWithExitCode 1
+  }
+
+  foreach ($tool in $ToolsList) {
+    # Put together the name and scenario to make a unique key.
+    $toolConfigName = $tool.Name
+    if ($tool.Scenario) {
+      $toolConfigName += "_" + $tool.Scenario
+    }
+
+    Write-Host "=== Configuring $toolConfigName..."
+
+    $gdnConfigFile = Join-Path $gdnConfigPath "$toolConfigName-configure.gdnconfig"
+
+    # For some tools, add default and automatic args.
+    if ($tool.Name -eq 'credscan') {
+      if ($targetDirectory) {
+        $tool.Args += "TargetDirectory < $TargetDirectory"
+      }
+      $tool.Args += "OutputType < pre"
+      $tool.Args += $CrScanAdditionalRunConfigParams
+    } elseif ($tool.Name -eq 'policheck') {
+      if ($targetDirectory) {
+        $tool.Args += "Target < $TargetDirectory"
+      }
+      $tool.Args += $PoliCheckAdditionalRunConfigParams
+    }
+
+    # Create variable pointing to the args array directly so we can use splat syntax later.
+    $toolArgs = $tool.Args
+
+    # Configure the tool. If args array is provided or the current tool has some default arguments
+    # defined, add "--args" and splat each element on the end. Arg format is "{Arg id} < {Value}",
+    # one per parameter. Doc page for "guardian configure":
+    # https://dev.azure.com/securitytools/SecurityIntegration/_wiki/wikis/Guardian/1395/configure
+    Exec-BlockVerbosely {
+      & $GuardianCliLocation configure `
+        --working-directory $WorkingDirectory `
+        --tool $tool.Name `
+        --output-path $gdnConfigFile `
+        --logger-level $GuardianLoggerLevel `
+        --noninteractive `
+        --force `
+        $(if ($toolArgs) { "--args" }) @toolArgs
+      Exit-IfNZEC "Sdl"
+    }
+
+    Write-Host "Created '$toolConfigName' configuration file: $gdnConfigFile"
+  }
+}
+catch {
+  Write-Host $_.ScriptStackTrace
+  Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
+  ExitWithExitCode 1
+}

eng/common/sdl/execute-all-sdl-tools.ps1

@@ -7,8 +7,17 @@ Param(
   [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY,                                         # Required: the directory where source files are located
   [string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ('artifacts')),  # Required: the directory where build artifacts are located
   [string] $AzureDevOpsAccessToken,                                                              # Required: access token for dnceng; should be provided via KeyVault
-  [string[]] $SourceToolsList,                                                                   # Optional: list of SDL tools to run on source code
-  [string[]] $ArtifactToolsList,                                                                 # Optional: list of SDL tools to run on built artifacts
+
+  # Optional: list of SDL tools to run on source code. See 'configure-sdl-tool.ps1' for tools list
+  # format.
+  [object[]] $SourceToolsList,
+  # Optional: list of SDL tools to run on built artifacts. See 'configure-sdl-tool.ps1' for tools
+  # list format.
+  [object[]] $ArtifactToolsList,
+  # Optional: list of SDL tools to run without automatically specifying a target directory. See
+  # 'configure-sdl-tool.ps1' for tools list format.
+  [object[]] $CustomToolsList,
+
   [bool] $TsaPublish=$False,                                                                     # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs.
   [string] $TsaBranchName=$env:BUILD_SOURCEBRANCH,                                               # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs.
   [string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME,                                        # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs.
@@ -63,13 +72,16 @@ try {
     ExitWithExitCode 1
   }
 
-  & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel
+  Exec-BlockVerbosely {
+    & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel
+  }
   $gdnFolder = Join-Path $workingDirectory '.gdn'
 
   if ($TsaOnboard) {
     if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) {
-      Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel"
-      & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
+      Exec-BlockVerbosely {
+        & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
+      }
       if ($LASTEXITCODE -ne 0) {
         Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-onboard failed with exit code $LASTEXITCODE."
         ExitWithExitCode $LASTEXITCODE
@@ -80,11 +92,41 @@ try {
     }
   }
 
-  if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) {
-    & $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
+  # Configure a list of tools with a default target directory. Populates the ".gdn/r" directory.
+  function Configure-ToolsList([object[]] $tools, [string] $targetDirectory) {
+    if ($tools -and $tools.Count -gt 0) {
+      Exec-BlockVerbosely {
+        & $(Join-Path $PSScriptRoot 'configure-sdl-tool.ps1') `
+          -GuardianCliLocation $guardianCliLocation `
+          -WorkingDirectory $workingDirectory `
+          -TargetDirectory $targetDirectory `
+          -GdnFolder $gdnFolder `
+          -ToolsList $tools `
+          -AzureDevOpsAccessToken $AzureDevOpsAccessToken `
+          -GuardianLoggerLevel $GuardianLoggerLevel `
+          -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams `
+          -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
+        if ($BreakOnFailure) {
+          Exit-IfNZEC "Sdl"
+        }
+      }
+    }
   }
-  if ($SourceToolsList -and $SourceToolsList.Count -gt 0) {
-    & $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
+
+  # Configure Artifact and Source tools with default Target directories.
+  Configure-ToolsList $ArtifactToolsList $ArtifactsDirectory
+  Configure-ToolsList $SourceToolsList $SourceDirectory
+  # Configure custom tools with no default Target directory.
+  Configure-ToolsList $CustomToolsList $null
+
+  # At this point, all tools are configured in the ".gdn" directory. Run them all in a single call.
+  # (If we used "run" multiple times, each run would overwrite data from earlier runs.)
+  Exec-BlockVerbosely {
+    & $(Join-Path $PSScriptRoot 'run-sdl.ps1') `
+      -GuardianCliLocation $guardianCliLocation `
+      -WorkingDirectory $workingDirectory `
+      -UpdateBaseline $UpdateBaseline `
+      -GdnFolder $gdnFolder
   }
 
   if ($TsaPublish) {
@@ -92,8 +134,9 @@ try {
       if (-not $TsaRepositoryName) {
         $TsaRepositoryName = "$($Repository)-$($BranchName)"
       }
-      Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel"
-      & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory  --logger-level $GuardianLoggerLevel
+      Exec-BlockVerbosely {
+        & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory  --logger-level $GuardianLoggerLevel
+      }
       if ($LASTEXITCODE -ne 0) {
         Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-publish failed with exit code $LASTEXITCODE."
         ExitWithExitCode $LASTEXITCODE
@@ -106,7 +149,11 @@ try {
 
   if ($BreakOnFailure) {
     Write-Host "Failing the build in case of breaking results..."
-    & $guardianCliLocation break
+    Exec-BlockVerbosely {
+      & $guardianCliLocation break --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
+    }
+  } else {
+    Write-Host "Letting the build pass even if there were breaking results..."
   }
 }
 catch {

eng/common/sdl/extract-artifact-archives.ps1

+# This script looks for each archive file in a directory and extracts it into the target directory.
+# For example, the file "$InputPath/bin.tar.gz" extracts to "$ExtractPath/bin.tar.gz.extracted/**".
+# Uses the "tar" utility added to Windows 10 / Windows 2019 that supports tar.gz and zip.
+param(
+  # Full path to directory where archives are stored.
+  [Parameter(Mandatory=$true)][string] $InputPath,
+  # Full path to directory to extract archives into. May be the same as $InputPath.
+  [Parameter(Mandatory=$true)][string] $ExtractPath
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+
+$disableConfigureToolsetImport = $true
+
+try {
+  # `tools.ps1` checks $ci to perform some actions. Since the SDL
+  # scripts don't necessarily execute in the same agent that run the
+  # build.ps1/sh script this variable isn't automatically set.
+  $ci = $true
+  . $PSScriptRoot\..\tools.ps1
+
+  Measure-Command {
+    $jobs = @()
+
+    # Find archive files for non-Windows and Windows builds.
+    $archiveFiles = @(
+      Get-ChildItem (Join-Path $InputPath "*.tar.gz")
+      Get-ChildItem (Join-Path $InputPath "*.zip")
+    )
+
+    foreach ($targzFile in $archiveFiles) {
+      $jobs += Start-Job -ScriptBlock {
+        $file = $using:targzFile
+        $fileName = [System.IO.Path]::GetFileName($file)
+        $extractDir = Join-Path $using:ExtractPath "$fileName.extracted"
+
+        New-Item $extractDir -ItemType Directory -Force | Out-Null
+
+        Write-Host "Extracting '$file' to '$extractDir'..."
+
+        # Pipe errors to stdout to prevent PowerShell detecting them and quitting the job early.
+        # This type of quit skips the catch, so we wouldn't be able to tell which file triggered the
+        # error. Save output so it can be stored in the exception string along with context.
+        $output = tar -xf $file -C $extractDir 2>&1
+        # Handle NZEC manually rather than using Exit-IfNZEC: we are in a background job, so we
+        # don't have access to the outer scope.
+        if ($LASTEXITCODE -ne 0) {
+          throw "Error extracting '$file': non-zero exit code ($LASTEXITCODE). Output: '$output'"
+        }
+
+        Write-Host "Extracted to $extractDir"
+      }
+    }
+
+    Receive-Job $jobs -Wait
+  }
+}
+catch {
+  Write-Host $_
+  Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
+  ExitWithExitCode 1
+}

eng/common/sdl/run-sdl.ps1

 Param(
   [string] $GuardianCliLocation,
   [string] $WorkingDirectory,
-  [string] $TargetDirectory,
   [string] $GdnFolder,
-  [string[]] $ToolsList,
   [string] $UpdateBaseline,
-  [string] $GuardianLoggerLevel='Standard',
-  [string[]] $CrScanAdditionalRunConfigParams,
-  [string[]] $PoliCheckAdditionalRunConfigParams
+  [string] $GuardianLoggerLevel='Standard'
 )
 
 $ErrorActionPreference = 'Stop'
@@ -23,7 +19,6 @@ try {
   . $PSScriptRoot\..\tools.ps1
 
   # We store config files in the r directory of .gdn
-  Write-Host $ToolsList
   $gdnConfigPath = Join-Path $GdnFolder 'r'
   $ValidPath = Test-Path $GuardianCliLocation
 
@@ -33,37 +28,18 @@ try {
     ExitWithExitCode 1
   }
 
-  $configParam = @('--config')
-
-  foreach ($tool in $ToolsList) {
-    $gdnConfigFile = Join-Path $gdnConfigPath "$tool-configure.gdnconfig"
-    Write-Host $tool
-    # We have to manually configure tools that run on source to look at the source directory only
-    if ($tool -eq 'credscan') {
-      Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory < $TargetDirectory `" `" OutputType < pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})"
-      & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory < $TargetDirectory " "OutputType < pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})
-      if ($LASTEXITCODE -ne 0) {
-        Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian configure for $tool failed with exit code $LASTEXITCODE."
-        ExitWithExitCode $LASTEXITCODE
-      }
-    }
-    if ($tool -eq 'policheck') {
-      Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target < $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})"
-      & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target < $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})
-      if ($LASTEXITCODE -ne 0) {
-        Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian configure for $tool failed with exit code $LASTEXITCODE."
-        ExitWithExitCode $LASTEXITCODE
-      }
-    }
-
-    $configParam+=$gdnConfigFile
-  }
-
-  Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam"
-  & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam
-  if ($LASTEXITCODE -ne 0) {
-    Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian run for $ToolsList using $configParam failed with exit code $LASTEXITCODE."
-    ExitWithExitCode $LASTEXITCODE
+  $gdnConfigFiles = Get-ChildItem $gdnConfigPath -Recurse -Include '*.gdnconfig'
+  Write-Host "Discovered Guardian config files:"
+  $gdnConfigFiles | Out-String | Write-Host
+
+  Exec-BlockVerbosely {
+    & $GuardianCliLocation run `
+      --working-directory $WorkingDirectory `
+      --baseline mainbaseline `
+      --update-baseline $UpdateBaseline `
+      --logger-level $GuardianLoggerLevel `
+      --config @gdnConfigFiles
+    Exit-IfNZEC "Sdl"
   }
 }
 catch {

eng/common/templates/job/execute-sdl.yml

@@ -2,17 +2,41 @@ parameters:
   enable: 'false'                                             # Whether the SDL validation job should execute or not
   overrideParameters: ''                                       # Optional: to override values for parameters.
   additionalParameters: ''                                     # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")'
+  # Optional: if specified, restore and use this version of Guardian instead of the default.
+  overrideGuardianVersion: ''
+  # Optional: if true, publish the '.gdn' folder as a pipeline artifact. This can help with in-depth
+  # diagnosis of problems with specific tool configurations.
+  publishGuardianDirectoryToPipeline: false
+  # The script to run to execute all SDL tools. Use this if you want to use a script to define SDL
+  # parameters rather than relying on YAML. It may be better to use a local script, because you can
+  # reproduce results locally without piecing together a command based on the YAML.
+  executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1'
   # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named
   # 'continueOnError', the parameter value is not correctly picked up.
   # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter
   sdlContinueOnError: false                                    # optional: determines whether to continue the build if the step errors;
-  downloadArtifacts: true                                      # optional: determines if the artifacts should be dowloaded
+  # optional: determines if build artifacts should be downloaded.
+  downloadArtifacts: true
+  # optional: determines if this job should search the directory of downloaded artifacts for
+  # 'tar.gz' and 'zip' archive files and extract them before running SDL validation tasks.
+  extractArchiveArtifacts: false
   dependsOn: ''                                                # Optional: dependencies of the job
   artifactNames: ''                                            # Optional: patterns supplied to DownloadBuildArtifacts
                                                                # Usage:
                                                                #  artifactNames:
                                                                #    - 'BlobArtifacts'
                                                                #    - 'Artifacts_Windows_NT_Release'
+  # Optional: download a list of pipeline artifacts. 'downloadArtifacts' controls build artifacts,
+  # not pipeline artifacts, so doesn't affect the use of this parameter.
+  pipelineArtifactNames: []
+  # Optional: location and ID of the AzDO build that the build/pipeline artifacts should be
+  # downloaded from. By default, uses runtime expressions to decide based on the variables set by
+  # the 'setupMaestroVars' dependency. Overriding this parameter is necessary if SDL tasks are
+  # running without Maestro++/BAR involved, or to download artifacts from a specific existing build
+  # to iterate quickly on SDL changes.
+  AzDOProjectName: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+  AzDOPipelineId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+  AzDOBuildId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
 
 jobs:
 - job: Run_SDL
@@ -22,16 +46,29 @@ jobs:
   variables:
     - group: DotNet-VSTS-Bot
     - name: AzDOProjectName
-      value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+      value: ${{ parameters.AzDOProjectName }}
     - name: AzDOPipelineId
-      value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+      value: ${{ parameters.AzDOPipelineId }}
     - name: AzDOBuildId
-      value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+      value: ${{ parameters.AzDOBuildId }}
+    # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in
+    # sync with the packages.config file.
+    - name: DefaultGuardianVersion
+      value: 0.53.3
+    - name: GuardianVersion
+      value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }}
+    - name: GuardianPackagesConfigFile
+      value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config
   pool:
-    name: Hosted VS2017
+    # To extract archives (.tar.gz, .zip), we need access to "tar", added in Windows 10/2019.
+    ${{ if eq(parameters.extractArchiveArtifacts, 'false') }}:
+      name: Hosted VS2017
+    ${{ if ne(parameters.extractArchiveArtifacts, 'false') }}:
+      vmImage: windows-2019
   steps:
   - checkout: self
     clean: true
+
   - ${{ if ne(parameters.downloadArtifacts, 'false')}}:
     - ${{ if ne(parameters.artifactNames, '') }}:
       - ${{ each artifactName in parameters.artifactNames }}:
@@ -59,16 +96,51 @@ jobs:
           itemPattern: "**"
           downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
           checkDownloadedFiles: true
+
+  - ${{ each artifactName in parameters.pipelineArtifactNames }}:
+    - task: DownloadPipelineArtifact@2
+      displayName: Download Pipeline Artifacts
+      inputs:
+        buildType: specific
+        buildVersionToDownload: specific
+        project: $(AzDOProjectName)
+        pipeline: $(AzDOPipelineId)
+        buildId: $(AzDOBuildId)
+        artifactName: ${{ artifactName }}
+        downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
+        checkDownloadedFiles: true
+
   - powershell: eng/common/sdl/extract-artifact-packages.ps1
       -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
       -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
     displayName: Extract Blob Artifacts
     continueOnError: ${{ parameters.sdlContinueOnError }}
+
   - powershell: eng/common/sdl/extract-artifact-packages.ps1
       -InputPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts
       -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts
     displayName: Extract Package Artifacts
     continueOnError: ${{ parameters.sdlContinueOnError }}
+
+  - ${{ if ne(parameters.extractArchiveArtifacts, 'false') }}:
+    - powershell: eng/common/sdl/extract-artifact-archives.ps1
+        -InputPath $(Build.ArtifactStagingDirectory)\artifacts
+        -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts
+      displayName: Extract Archive Artifacts
+      continueOnError: ${{ parameters.sdlContinueOnError }}
+  
+  - ${{ if ne(parameters.overrideGuardianVersion, '') }}:
+    - powershell: |
+        $content = Get-Content $(GuardianPackagesConfigFile)
+
+        Write-Host "packages.config content was:`n$content"
+
+        $content = $content.Replace('$(DefaultGuardianVersion)', '$(GuardianVersion)')
+        $content | Set-Content $(GuardianPackagesConfigFile)
+
+        Write-Host "packages.config content updated to:`n$content"
+      displayName: Use overridden Guardian version ${{ parameters.overrideGuardianVersion }}
+
   - task: NuGetToolInstaller@1
     displayName: 'Install NuGet.exe'
   - task: NuGetCommand@2
@@ -79,15 +151,35 @@ jobs:
       nugetConfigPath: $(Build.SourcesDirectory)\eng\common\sdl\NuGet.config
       externalFeedCredentials: GuardianConnect
       restoreDirectory: $(Build.SourcesDirectory)\.packages
+
   - ${{ if ne(parameters.overrideParameters, '') }}:
-    - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 ${{ parameters.overrideParameters }}
+    - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }}
       displayName: Execute SDL
       continueOnError: ${{ parameters.sdlContinueOnError }}
   - ${{ if eq(parameters.overrideParameters, '') }}:
-    - powershell: eng/common/sdl/execute-all-sdl-tools.ps1
-        -GuardianPackageName Microsoft.Guardian.Cli.0.53.3
+    - powershell: ${{ parameters.executeAllSdlToolsScript }}
+        -GuardianPackageName Microsoft.Guardian.Cli.$(GuardianVersion)
         -NugetPackageDirectory $(Build.SourcesDirectory)\.packages
         -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw)
         ${{ parameters.additionalParameters }}
       displayName: Execute SDL
       continueOnError: ${{ parameters.sdlContinueOnError }}
+
+  - ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}:
+    # We want to publish the Guardian results and configuration for easy diagnosis. However, the
+    # '.gdn' dir is a mix of configuration, results, extracted dependencies, and Guardian default
+    # tooling files. Some of these files are large and aren't useful during an investigation, so
+    # exclude them by simply deleting them before publishing. (As of writing, there is no documented
+    # way to selectively exclude a dir from the pipeline artifact publish task.)
+    - task: DeleteFiles@1
+      displayName: Delete Guardian dependencies to avoid uploading
+      inputs:
+        SourceFolder: $(Agent.BuildDirectory)/.gdn
+        Contents: |
+          c
+          i
+      condition: succeededOrFailed()
+    - publish: $(Agent.BuildDirectory)/.gdn
+      artifact: GuardianConfiguration
+      displayName: Publish GuardianConfiguration
+      condition: succeededOrFailed()

eng/common/tools.ps1

@@ -42,7 +42,7 @@
 [bool]$useInstalledDotNetCli = if (Test-Path variable:useInstalledDotNetCli) { $useInstalledDotNetCli } else { $true }
 
 # Enable repos to use a particular version of the on-line dotnet-install scripts.
-#    default URL: https://dot.net/v1/dotnet-install.ps1
+#    default URL: https://dotnet.microsoft.com/download/dotnet/scripts/v1/dotnet-install.ps1
 [string]$dotnetInstallScriptVersion = if (Test-Path variable:dotnetInstallScriptVersion) { $dotnetInstallScriptVersion } else { 'v1' }
 
 # True to use global NuGet cache instead of restoring packages to repository-local directory.
@@ -106,6 +106,46 @@ function Exec-Process([string]$command, [string]$commandArgs) {
   }
 }
 
+# Take the given block, print it, print what the block probably references from the current set of
+# variables using low-effort string matching, then run the block.
+#
+# This is intended to replace the pattern of manually copy-pasting a command, wrapping it in quotes,
+# and printing it using "Write-Host". The copy-paste method is more readable in build logs, but less
+# maintainable and less reliable. It is easy to make a mistake and modify the command without
+# properly updating the "Write-Host" line, resulting in misleading build logs. The probability of
+# this mistake makes the pattern hard to trust when it shows up in build logs. Finding the bug in
+# existing source code can also be difficult, because the strings are not aligned to each other and
+# the line may be 300+ columns long.
+#
+# By removing the need to maintain two copies of the command, Exec-BlockVerbosely avoids the issues.
+#
+# In Bash (or any posix-like shell), "set -x" prints usable verbose output automatically.
+# "Set-PSDebug" appears to be similar at first glance, but unfortunately, it isn't very useful: it
+# doesn't print any info about the variables being used by the command, which is normally the
+# interesting part to diagnose.
+function Exec-BlockVerbosely([scriptblock] $block) {
+  Write-Host "--- Running script block:"
+  $blockString = $block.ToString().Trim()
+  Write-Host $blockString
+
+  Write-Host "--- List of variables that might be used:"
+  # For each variable x in the environment, check the block for a reference to x via simple "$x" or
+  # "@x" syntax. This doesn't detect other ways to reference variables ("${x}" nor "$variable:x",
+  # among others). It only catches what this function was originally written for: simple
+  # command-line commands.
+  $variableTable = Get-Variable |
+    Where-Object {
+      $blockString.Contains("`$$($_.Name)") -or $blockString.Contains("@$($_.Name)")
+    } |
+    Format-Table -AutoSize -HideTableHeaders -Wrap |
+    Out-String
+  Write-Host $variableTable.Trim()
+
+  Write-Host "--- Executing:"
+  & $block
+  Write-Host "--- Done running script block!"
+}
+
 # createSdkLocationFile parameter enables a file being generated under the toolset directory
 # which writes the sdk's location into. This is only necessary for cmd --> powershell invocations
 # as dot sourcing isn't possible.
@@ -223,7 +263,7 @@ function GetDotNetInstallScript([string] $dotnetRoot) {
   if (!(Test-Path $installScript)) {
     Create-Directory $dotnetRoot
     $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
-    $uri = "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1"
+    $uri = "https://dotnet.microsoft.com/download/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.ps1"
 
     Retry({
       Write-Host "GET $uri"
@@ -632,6 +672,17 @@ function ExitWithExitCode([int] $exitCode) {
   exit $exitCode
 }
 
+# Check if $LASTEXITCODE is a nonzero exit code (NZEC). If so, print a Azure Pipeline error for
+# diagnostics, then exit the script with the $LASTEXITCODE.
+function Exit-IfNZEC([string] $category = "General") {
+  Write-Host "Exit code $LASTEXITCODE"
+  if ($LASTEXITCODE -ne 0) {
+    $message = "Last command failed with exit code $LASTEXITCODE."
+    Write-PipelineTelemetryError -Force -Category $category -Message $message
+    ExitWithExitCode $LASTEXITCODE
+  }
+}
+
 function Stop-Processes() {
   Write-Host 'Killing running build processes...'
   foreach ($processName in $processesToStopOnExit) {

eng/common/tools.sh

@@ -54,7 +54,7 @@ warn_as_error=${warn_as_error:-true}
 use_installed_dotnet_cli=${use_installed_dotnet_cli:-true}
 
 # Enable repos to use a particular version of the on-line dotnet-install scripts.
-#    default URL: https://dot.net/v1/dotnet-install.sh
+#    default URL: https://dotnet.microsoft.com/download/dotnet/scripts/v1/dotnet-install.sh
 dotnetInstallScriptVersion=${dotnetInstallScriptVersion:-'v1'}
 
 # True to use global NuGet cache instead of restoring packages to repository-local directory.
@@ -262,7 +262,7 @@ function with_retries {
 function GetDotNetInstallScript {
   local root=$1
   local install_script="$root/dotnet-install.sh"
-  local install_script_url="https://dot.net/$dotnetInstallScriptVersion/dotnet-install.sh"
+  local install_script_url="https://dotnet.microsoft.com/download/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.sh"
 
   if [[ ! -a "$install_script" ]]; then
     mkdir -p "$root"

global.json

@@ -12,7 +12,7 @@
     "version": "6.0.100-preview.4.21255.9"
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21330.2",
-    "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21330.2"
+    "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21364.3",
+    "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21364.3"
   }
 }