Skip to content

R
runtime

dotNET Platform / runtime
11 个月 前同步成功

通知 1
Star 1
Fork 0
R
runtime

体验新版 GitCode，发现更多精彩内容 >>

未验证 提交 cdff28e8 编写于 作者: V Viktor Hofer 提交者： GitHub
浏览文件
操作

Delete publish-final.proj and clean-up installer.tasks unused files (#48598) 

* Delete publish-final.proj and delete unused installer.tasks files

* Avoid extra restore phase for tasks

The repo local tasks don't participate in the repo restore anymore,
hence avoiding the extra restore phase which should improve reliability
and build times.
上级 f23f0167
隐藏空白更改
内联 并排
Showing with 10 addition and 2083 deletion
Build.proj
浏览文件 @ cdff28e8
......@@ -4,12 +4,11 @@
<TraversalGlobalProperties Condition="'$(BuildAllConfigurations)' != 'true'">BuildTargetFramework=$([MSBuild]::ValueOrDefault('$(BuildTargetFramework)', '$(NetCoreAppCurrent)'))</TraversalGlobalProperties>
</PropertyGroup>
<!--
Subsets are already imported by Directory.Build.props.
Reference the projects for traversal build. Ordering matters here.
-->
<ItemGroup>
<!-- Subsets are already imported by Directory.Build.props. -->
<ProjectReference Include="@(ProjectToBuild)" />
<!-- Only include tasks.proj during restore and build incrementally via a target. -->
<ProjectReference Include="$(RepoTasksDir)tasks.proj" Condition="'$(MSBuildRestoreSessionId)' != ''" />
</ItemGroup>
<!-- Custom arcade target which isn't available in Microsoft.Build.Traversal. -->
......@@ -22,8 +21,8 @@
<Import Project="$(RepositoryEngineeringDir)restore\optimizationData.targets" Condition="'$(DotNetBuildFromSource)' != 'true' and '$(EnableNgenOptimization)' == 'true'" />
<Target Name="BuildLocalTasks"
BeforeTargets="Restore">
BeforeTargets="Build">
<MSBuild Projects="$(RepoTasksDir)tasks.proj"
Targets="BuildAndRestoreIncrementally"/>
Targets="BuildIncrementally" />
</Target>
</Project>
\ No newline at end of file
Directory.Build.props
浏览文件 @ cdff28e8
......@@ -70,7 +70,7 @@
<WasmBuildTasksDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'WasmBuildTasks', 'Debug', '$(NetCoreAppToolCurrent)', 'publish'))</WasmBuildTasksDir>
<MonoAOTCompilerDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'MonoAOTCompiler', 'Debug', '$(NetCoreAppToolCurrent)'))</MonoAOTCompilerDir>
<InstallerTasksAssemblyPath Condition="'$(MSBuildRuntimeType)' == 'Core'">$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'installer.tasks', 'Debug', 'netstandard2.0', 'installer.tasks.dll'))</InstallerTasksAssemblyPath>
<InstallerTasksAssemblyPath Condition="'$(MSBuildRuntimeType)' == 'Core'">$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'installer.tasks', 'Debug', '$(NetCoreAppToolCurrent)', 'installer.tasks.dll'))</InstallerTasksAssemblyPath>
<InstallerTasksAssemblyPath Condition="'$(MSBuildRuntimeType)' != 'Core'">$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'installer.tasks', 'Debug', 'net461', 'installer.tasks.dll'))</InstallerTasksAssemblyPath>
<AppleAppBuilderTasksAssemblyPath>$([MSBuild]::NormalizePath('$(AppleAppBuilderDir)', 'AppleAppBuilder.dll'))</AppleAppBuilderTasksAssemblyPath>
<AndroidAppBuilderTasksAssemblyPath>$([MSBuild]::NormalizePath('$(AndroidAppBuilderDir)', 'AndroidAppBuilder.dll'))</AndroidAppBuilderTasksAssemblyPath>
......
src/installer/publish/publish-final.proj 已删除 100644 → 0
浏览文件 @ f23f0167
<Project>
<Import Project="Sdk.props" Sdk="Microsoft.NET.Sdk" />
<UsingTask TaskName="FinalizeBuild" AssemblyFile="$(InstallerTasksAssemblyPath)" />
<!--
Target wrapping UpdatePublishedVersions: ensures that ShippedNuGetPackage items are created and
disables versions repo update if no auth token is defined. Otherwise, not specifying an auth
token would cause an error.
-->
<Target Name="UpdateVersionsRepo"
Condition="'$(GitHubAuthToken)' != ''"
DependsOnTargets="
GetShippedNuGetPackages;
UpdatePublishedVersions" />
<Target Name="GetShippedNuGetPackages"
DependsOnTargets="FindDownloadedArtifacts">
<ItemGroup>
<!-- Nupkgs to include in dotnet/versions update. -->
<ShippedNuGetPackage Include="@(NupkgToPublishFile)" />
</ItemGroup>
<Message Importance="High" Text="Updating versions repo '$(VersionsRepoOwner)/$(VersionsRepo)' path '$(VersionsRepoPath)' with nupkgs:" />
<Message Importance="High" Text="%(NupkgToPublishFile.Filename)" />
</Target>
<PropertyGroup>
<FinalizeBuildInAzureDependsOn Condition="'$(DisableSourceLink)' != 'true'">InitializeSourceControlInformationFromSourceControlManager</FinalizeBuildInAzureDependsOn>
</PropertyGroup>
<Target Name="FinalizeBuildInAzure"
DependsOnTargets="GetProductVersions;$(FinalizeBuildInAzureDependsOn)">
<Error Condition="'$(AzureAccessToken)' == ''" Text="Missing required property 'AzureAccessToken'" />
<Error Condition="'$(AzureAccountName)' == ''" Text="Missing required property 'AzureAccountName'" />
<Error Condition="'$(ContainerName)' == ''" Text="Missing required property 'ContainerName'" />
<Message Importance="High" Text="Finalizing Build version $(ProductVersion) to channel $(Channel)" />
<FinalizeBuild
AccountName="$(AzureAccountName)"
AccountKey="$(AzureAccessToken)"
ContainerName="$(ContainerName)"
SemaphoreBlob="Runtime/$(Channel)/sharedFxPublishSemaphore"
Channel="$(Channel)"
Version="$(SharedFrameworkNugetVersion)"
SharedFrameworkNugetVersion="$(SharedFrameworkNugetVersion)"
SharedHostNuGetVersion="$(HostVersion)"
ProductVersion="$(ProductVersion)"
CommitHash="$([MSBuild]::ValueOrDefault('$(SourceRevisionId)', 'N/A'))"
FinalizeContainer="Runtime/$(SharedFrameworkNugetVersion)"
ForcePublish="true" />
<Error Condition="'$(ChecksumAzureAccessToken)' == ''" Text="Missing required property 'ChecksumAzureAccessToken'" />
<Error Condition="'$(ChecksumAzureAccountName)' == ''" Text="Missing required property 'ChecksumAzureAccountName'" />
<Error Condition="'$(ChecksumContainerName)' == ''" Text="Missing required property 'ChecksumContainerName'" />
<Message Importance="High" Text="Finalizing Checksums" />
<FinalizeBuild
AccountName="$(ChecksumAzureAccountName)"
AccountKey="$(ChecksumAzureAccessToken)"
ContainerName="$(ChecksumContainerName)"
SemaphoreBlob="Runtime/$(Channel)/checksumPublishSemaphore"
Channel="$(Channel)"
Version="$(SharedFrameworkNugetVersion)"
SharedFrameworkNugetVersion="$(SharedFrameworkNugetVersion)"
SharedHostNuGetVersion="$(HostVersion)"
ProductVersion="$(ProductVersion)"
CommitHash="$([MSBuild]::ValueOrDefault('$(SourceRevisionId)', 'N/A'))"
FinalizeContainer="Runtime/$(SharedFrameworkNugetVersion)"
ForcePublish="true" />
</Target>
<Import Project="Sdk.targets" Sdk="Microsoft.NET.Sdk" />
<!-- Perform extra publish steps that Arcade doesn't support. -->
<Target Name="Build"
DependsOnTargets="
FinalizeBuildInAzure;
UpdateVersionsRepo">
<Message Importance="High" Text="Complete!" />
</Target>
</Project>
src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/AzureBlobLease.cs 已删除 100644 → 0
浏览文件 @ f23f0167
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
using Microsoft.Build.Framework;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
namespace Microsoft.DotNet.Build.CloudTestTasks
{
public partial class AzureBlobLease
{
private string _containerName;
private string _blobName;
private TimeSpan _maxWait;
private TimeSpan _delay;
private const int s_MaxWaitDefault = 60; // seconds
private const int s_DelayDefault = 500; // milliseconds
private CancellationTokenSource _cancellationTokenSource;
private Task _leaseRenewalTask;
private string _connectionString;
private string _accountName;
private string _accountKey;
private Microsoft.Build.Utilities.TaskLoggingHelper _log;
private string _leaseId;
private string _leaseUrl;
public AzureBlobLease(string accountName, string accountKey, string connectionString, string containerName, string blobName, Microsoft.Build.Utilities.TaskLoggingHelper log, string maxWait = null, string delay = null)
{
_accountName = accountName;
_accountKey = accountKey;
_connectionString = connectionString;
_containerName = containerName;
_blobName = blobName;
_maxWait = !string.IsNullOrWhiteSpace(maxWait) ? TimeSpan.Parse(maxWait) : TimeSpan.FromSeconds(s_MaxWaitDefault);
_delay = !string.IsNullOrWhiteSpace(delay) ? TimeSpan.Parse(delay) : TimeSpan.FromMilliseconds(s_DelayDefault);
_log = log;
_leaseUrl = $"{AzureHelper.GetBlobRestUrl(_accountName, _containerName, _blobName)}?comp=lease";
}
public string Acquire()
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
while (stopWatch.ElapsedMilliseconds < _maxWait.TotalMilliseconds)
{
try
{
string leaseId = AcquireLeaseOnBlobAsync().GetAwaiter().GetResult();
_cancellationTokenSource = new CancellationTokenSource();
_leaseRenewalTask = Task.Run(() =>
{ AutoRenewLeaseOnBlob(this, _log); },
_cancellationTokenSource.Token);
_leaseId = leaseId;
return _leaseId;
}
catch (Exception e)
{
_log.LogMessage($"Retrying lease acquisition on {_blobName}, {e.Message}");
Thread.Sleep(_delay);
}
}
ResetLeaseRenewalTaskState();
throw new Exception($"Unable to acquire lease on {_blobName}");
}
public void Release()
{
// Cancel the lease renewal task since we are about to release the lease.
ResetLeaseRenewalTaskState();
using (HttpClient client = new HttpClient())
{
Tuple<string, string> leaseAction = new Tuple<string, string>("x-ms-lease-action", "release");
Tuple<string, string> headerLeaseId = new Tuple<string, string>("x-ms-lease-id", _leaseId);
List<Tuple<string, string>> additionalHeaders = new List<Tuple<string, string>>() { leaseAction, headerLeaseId };
var request = AzureHelper.RequestMessage("PUT", _leaseUrl, _accountName, _accountKey, additionalHeaders);
using (HttpResponseMessage response = AzureHelper.RequestWithRetry(_log, client, request).GetAwaiter().GetResult())
{
if (!response.IsSuccessStatusCode)
{
_log.LogMessage($"Unable to release lease on container/blob {_containerName}/{_blobName}.");
}
}
}
}
private async Task<string> AcquireLeaseOnBlobAsync()
{
_log.LogMessage(MessageImportance.Low, $"Requesting lease for container/blob '{_containerName}/{_blobName}'.");
string leaseId = string.Empty;
using (HttpClient client = new HttpClient())
{
try
{
Tuple<string, string> leaseAction = new Tuple<string, string>("x-ms-lease-action", "acquire");
Tuple<string, string> leaseDuration = new Tuple<string, string>("x-ms-lease-duration", "60" /* seconds */);
List<Tuple<string, string>> additionalHeaders = new List<Tuple<string, string>>() { leaseAction, leaseDuration };
var request = AzureHelper.RequestMessage("PUT", _leaseUrl, _accountName, _accountKey, additionalHeaders);
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(_log, client, request))
{
leaseId = response.Headers.GetValues("x-ms-lease-id").FirstOrDefault();
}
}
catch (Exception e)
{
_log.LogErrorFromException(e, true);
}
}
return leaseId;
}
private static void AutoRenewLeaseOnBlob(AzureBlobLease instance, Microsoft.Build.Utilities.TaskLoggingHelper log)
{
TimeSpan maxWait = TimeSpan.FromSeconds(s_MaxWaitDefault);
TimeSpan delay = TimeSpan.FromMilliseconds(s_DelayDefault);
TimeSpan waitFor = maxWait;
CancellationToken token = instance._cancellationTokenSource.Token;
while (true)
{
token.ThrowIfCancellationRequested();
try
{
log.LogMessage(MessageImportance.Low, $"Requesting lease for container/blob '{instance._containerName}/{instance._blobName}'.");
using (HttpClient client = new HttpClient())
{
Tuple<string, string> leaseAction = new Tuple<string, string>("x-ms-lease-action", "renew");
Tuple<string, string> headerLeaseId = new Tuple<string, string>("x-ms-lease-id", instance._leaseId);
List<Tuple<string, string>> additionalHeaders = new List<Tuple<string, string>>() { leaseAction, headerLeaseId };
var request = AzureHelper.RequestMessage("PUT", instance._leaseUrl, instance._accountName, instance._accountKey, additionalHeaders);
using (HttpResponseMessage response = AzureHelper.RequestWithRetry(log, client, request).GetAwaiter().GetResult())
{
if (!response.IsSuccessStatusCode)
{
throw new Exception("Unable to acquire lease.");
}
}
}
waitFor = maxWait;
}
catch (Exception e)
{
Console.WriteLine($"Rerying lease renewal on {instance._containerName}, {e.Message}");
waitFor = delay;
}
token.ThrowIfCancellationRequested();
Task.Delay(waitFor, token).Wait();
}
}
private void ResetLeaseRenewalTaskState()
{
// Cancel the lease renewal task if it was created
if (_leaseRenewalTask != null)
{
_cancellationTokenSource.Cancel();
// Block until the task ends. It can throw if we cancelled it before it completed.
try
{
_leaseRenewalTask.Wait();
}
catch (Exception)
{
// Ignore the caught exception as it will be expected.
}
_leaseRenewalTask = null;
}
}
}
}
src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/AzureConnectionStringBuildTask.cs 已删除 100644 → 0
浏览文件 @ f23f0167
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
using Microsoft.Build.Utilities;
using System.Text.RegularExpressions;
namespace Microsoft.DotNet.Build.CloudTestTasks
{
public abstract class AzureConnectionStringBuildTask : Task
{
/// <summary>
/// Azure Storage account connection string. Supersedes Account Key / Name.
/// Will cause errors if both are set.
/// </summary>
public string ConnectionString { get; set; }
/// <summary>
/// The Azure account key used when creating the connection string.
/// When we fully deprecate these, can just make them get; only.
/// </summary>
public string AccountKey { get; set; }
/// <summary>
/// The Azure account name used when creating the connection string.
/// When we fully deprecate these, can just make them get; only.
/// </summary>
public string AccountName { get; set; }
public void ParseConnectionString()
{
if (!string.IsNullOrEmpty(ConnectionString))
{
if (!(string.IsNullOrEmpty(AccountKey) && string.IsNullOrEmpty(AccountName)))
{
Log.LogError("If the ConnectionString property is set, you must not provide AccountKey / AccountName. These values will be deprecated in the future.");
}
else
{
Regex storageConnectionStringRegex = new Regex("AccountName=(?<name>.+?);AccountKey=(?<key>.+?);");
MatchCollection matches = storageConnectionStringRegex.Matches(ConnectionString);
if (matches.Count > 0)
{
// When we deprecate this format, we'll want to demote these to private
AccountName = matches[0].Groups["name"].Value;
AccountKey = matches[0].Groups["key"].Value;
}
else
{
Log.LogError("Error parsing connection string. Please review its value.");
}
}
}
else if (string.IsNullOrEmpty(AccountKey) || string.IsNullOrEmpty(AccountName))
{
Log.LogError("Error, must provide either ConnectionString or AccountName with AccountKey");
}
}
}
}
src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/AzureHelper.cs 已删除 100644 → 0
浏览文件 @ f23f0167
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
namespace Microsoft.DotNet.Build.CloudTestTasks
{
public static class AzureHelper
{
/// <summary>
/// The storage api version.
/// </summary>
public static readonly string StorageApiVersion = "2015-04-05";
public const string DateHeaderString = "x-ms-date";
public const string VersionHeaderString = "x-ms-version";
public const string AuthorizationHeaderString = "Authorization";
public const string CacheControlString = "x-ms-blob-cache-control";
public const string ContentTypeString = "x-ms-blob-content-type";
public enum SasAccessType
{
Read,
Write,
};
public static string AuthorizationHeader(
string storageAccount,
string storageKey,
string method,
DateTime now,
HttpRequestMessage request,
string ifMatch = "",
string contentMD5 = "",
string size = "",
string contentType = "")
{
string stringToSign = string.Format(
"{0}\n\n\n{1}\n{5}\n{6}\n\n\n{2}\n\n\n\n{3}{4}",
method,
(size == string.Empty) ? string.Empty : size,
ifMatch,
GetCanonicalizedHeaders(request),
GetCanonicalizedResource(request.RequestUri, storageAccount),
contentMD5,
contentType);
byte[] signatureBytes = Encoding.UTF8.GetBytes(stringToSign);
string authorizationHeader;
using (HMACSHA256 hmacsha256 = new HMACSHA256(Convert.FromBase64String(storageKey)))
{
authorizationHeader = "SharedKey " + storageAccount + ":"
+ Convert.ToBase64String(hmacsha256.ComputeHash(signatureBytes));
}
return authorizationHeader;
}
public static string CreateContainerSasToken(
string accountName,
string containerName,
string key,
SasAccessType accessType,
int validityTimeInDays)
{
string signedPermissions = string.Empty;
switch (accessType)
{
case SasAccessType.Read:
signedPermissions = "r";
break;
case SasAccessType.Write:
signedPermissions = "wdl";
break;
default:
throw new ArgumentOutOfRangeException(nameof(accessType), accessType, "Unrecognized value");
}
string signedStart = DateTime.UtcNow.ToString("O");
string signedExpiry = DateTime.UtcNow.AddDays(validityTimeInDays).ToString("O");
string canonicalizedResource = "/blob/" + accountName + "/" + containerName;
string signedIdentifier = string.Empty;
string signedVersion = StorageApiVersion;
string stringToSign = ConstructServiceStringToSign(
signedPermissions,
signedVersion,
signedExpiry,
canonicalizedResource,
signedIdentifier,
signedStart);
byte[] signatureBytes = Encoding.UTF8.GetBytes(stringToSign);
string signature;
using (HMACSHA256 hmacSha256 = new HMACSHA256(Convert.FromBase64String(key)))
{
signature = Convert.ToBase64String(hmacSha256.ComputeHash(signatureBytes));
}
string sasToken = string.Format(
"?sv={0}&sr={1}&sig={2}&st={3}&se={4}&sp={5}",
WebUtility.UrlEncode(signedVersion),
WebUtility.UrlEncode("c"),
WebUtility.UrlEncode(signature),
WebUtility.UrlEncode(signedStart),
WebUtility.UrlEncode(signedExpiry),
WebUtility.UrlEncode(signedPermissions));
return sasToken;
}
public static string GetCanonicalizedHeaders(HttpRequestMessage request)
{
StringBuilder sb = new StringBuilder();
List<string> headerNameList = (from headerName in request.Headers
where
headerName.Key.ToLowerInvariant()
.StartsWith("x-ms-", StringComparison.Ordinal)
select headerName.Key.ToLowerInvariant()).ToList();
headerNameList.Sort();
foreach (string headerName in headerNameList)
{
StringBuilder builder = new StringBuilder(headerName);
string separator = ":";
foreach (string headerValue in GetHeaderValues(request.Headers, headerName))
{
string trimmedValue = headerValue.Replace("\r\n", string.Empty);
builder.Append(separator);
builder.Append(trimmedValue);
separator = ",";
}
sb.Append(builder);
sb.Append("\n");
}
return sb.ToString();
}
public static string GetCanonicalizedResource(Uri address, string accountName)
{
StringBuilder str = new StringBuilder();
StringBuilder builder = new StringBuilder("/");
builder.Append(accountName);
builder.Append(address.AbsolutePath);
str.Append(builder);
Dictionary<string, HashSet<string>> queryKeyValues = ExtractQueryKeyValues(address);
Dictionary<string, HashSet<string>> dictionary = GetCommaSeparatedList(queryKeyValues);
foreach (KeyValuePair<string, HashSet<string>> pair in dictionary.OrderBy(p => p.Key))
{
StringBuilder stringBuilder = new StringBuilder(string.Empty);
stringBuilder.Append(pair.Key + ":");
string commaList = string.Join(",", pair.Value);
stringBuilder.Append(commaList);
str.Append("\n");
str.Append(stringBuilder);
}
return str.ToString();
}
public static List<string> GetHeaderValues(HttpRequestHeaders headers, string headerName)
{
List<string> list = new List<string>();
IEnumerable<string> values;
headers.TryGetValues(headerName, out values);
if (values != null)
{
list.Add((values.FirstOrDefault() ?? string.Empty).TrimStart(null));
}
return list;
}
private static bool IsWithinRetryRange(HttpStatusCode statusCode)
{
// Retry on http client and server error codes (4xx - 5xx) as well as redirect
var rawStatus = (int)statusCode;
if (rawStatus == 302)
return true;
else if (rawStatus >= 400 && rawStatus <= 599)
return true;
else
return false;
}
public static async Task<HttpResponseMessage> RequestWithRetry(TaskLoggingHelper loggingHelper, HttpClient client,
Func<HttpRequestMessage> createRequest, Func<HttpResponseMessage, bool> validationCallback = null, int retryCount = 5,
int retryDelaySeconds = 5)
{
if (loggingHelper == null)
throw new ArgumentNullException(nameof(loggingHelper));
if (client == null)
throw new ArgumentNullException(nameof(client));
if (createRequest == null)
throw new ArgumentNullException(nameof(createRequest));
if (retryCount < 1)
throw new ArgumentException(nameof(retryCount));
if (retryDelaySeconds < 1)
throw new ArgumentException(nameof(retryDelaySeconds));
int retries = 0;
HttpResponseMessage response = null;
// add a bit of randomness to the retry delay
var rng = new Random();
while (retries < retryCount)
{
if (retries > 0)
{
if (response != null)
{
response.Dispose();
response = null;
}
int delay = retryDelaySeconds * retries * rng.Next(1, 5);
loggingHelper.LogMessage(MessageImportance.Low, "Waiting {0} seconds before retry", delay);
await System.Threading.Tasks.Task.Delay(delay * 1000);
}
try
{
using (var request = createRequest())
response = await client.SendAsync(request);
}
catch (Exception e)
{
loggingHelper.LogWarningFromException(e, true);
// if this is the final iteration let the exception bubble up
if (retries + 1 == retryCount)
throw;
}
// response can be null if we fail to send the request
if (response != null)
{
if (validationCallback == null)
{
// check if the response code is within the range of failures
if (!IsWithinRetryRange(response.StatusCode))
{
return response;
}
}
else
{
bool isSuccess = validationCallback(response);
if (!isSuccess)
{
loggingHelper.LogMessage("Validation callback returned retry for status code {0}", response.StatusCode);
}
else
{
loggingHelper.LogMessage("Validation callback returned success for status code {0}", response.StatusCode);
return response;
}
}
}
++retries;
}
// retry count exceeded
loggingHelper.LogWarning("Retry count {0} exceeded", retryCount);
// set some default values in case response is null
var statusCode = "None";
var contentStr = "Null";
if (response != null)
{
statusCode = response.StatusCode.ToString();
contentStr = await response.Content.ReadAsStringAsync();
response.Dispose();
}
throw new HttpRequestException($"Request {createRequest().RequestUri} failed with status {statusCode}. Response : {contentStr}");
}
private static string ConstructServiceStringToSign(
string signedPermissions,
string signedVersion,
string signedExpiry,
string canonicalizedResource,
string signedIdentifier,
string signedStart,
string signedIP = "",
string signedProtocol = "",
string rscc = "",
string rscd = "",
string rsce = "",
string rscl = "",
string rsct = "")
{
// constructing string to sign based on spec in https://msdn.microsoft.com/en-us/library/azure/dn140255.aspx
var stringToSign = string.Join(
"\n",
signedPermissions,
signedStart,
signedExpiry,
canonicalizedResource,
signedIdentifier,
signedIP,
signedProtocol,
signedVersion,
rscc,
rscd,
rsce,
rscl,
rsct);
return stringToSign;
}
private static Dictionary<string, HashSet<string>> ExtractQueryKeyValues(Uri address)
{
Dictionary<string, HashSet<string>> values = new Dictionary<string, HashSet<string>>();
//Decode this to allow the regex to pull out the correct groups for signing
address = new Uri(WebUtility.UrlDecode(address.ToString()));
Regex newreg = new Regex(@"(?:\?|&)([^=]+)=([^&]+)");
MatchCollection matches = newreg.Matches(address.Query);
foreach (Match match in matches)
{
string key, value;
if (!string.IsNullOrEmpty(match.Groups[1].Value))
{
key = match.Groups[1].Value;
value = match.Groups[2].Value;
}
else
{
key = match.Groups[3].Value;
value = match.Groups[4].Value;
}
HashSet<string> setOfValues;
if (values.TryGetValue(key, out setOfValues))
{
setOfValues.Add(value);
}
else
{
HashSet<string> newSet = new HashSet<string> { value };
values.Add(key, newSet);
}
}
return values;
}
private static Dictionary<string, HashSet<string>> GetCommaSeparatedList(
Dictionary<string, HashSet<string>> queryKeyValues)
{
Dictionary<string, HashSet<string>> dictionary = new Dictionary<string, HashSet<string>>();
foreach (string queryKeys in queryKeyValues.Keys)
{
HashSet<string> setOfValues;
queryKeyValues.TryGetValue(queryKeys, out setOfValues);
List<string> list = new List<string>();
list.AddRange(setOfValues);
list.Sort();
string commaSeparatedValues = string.Join(",", list);
string key = queryKeys.ToLowerInvariant();
HashSet<string> setOfValues2;
if (dictionary.TryGetValue(key, out setOfValues2))
{
setOfValues2.Add(commaSeparatedValues);
}
else
{
HashSet<string> newSet = new HashSet<string> { commaSeparatedValues };
dictionary.Add(key, newSet);
}
}
return dictionary;
}
public static Func<HttpRequestMessage> RequestMessage(string method, string url, string accountName, string accountKey, List<Tuple<string, string>> additionalHeaders = null, string body = null)
{
Func<HttpRequestMessage> requestFunc = () =>
{
HttpMethod httpMethod = HttpMethod.Get;
if (method == "PUT")
{
httpMethod = HttpMethod.Put;
}
else if (method == "DELETE")
{
httpMethod = HttpMethod.Delete;
}
DateTime dateTime = DateTime.UtcNow;
var request = new HttpRequestMessage(httpMethod, url);
request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture));
request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
if (additionalHeaders != null)
{
foreach (Tuple<string, string> additionalHeader in additionalHeaders)
{
request.Headers.Add(additionalHeader.Item1, additionalHeader.Item2);
}
}
if (body != null)
{
request.Content = new StringContent(body);
request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
accountName,
accountKey,
method,
dateTime,
request,
"",
"",
request.Content.Headers.ContentLength.ToString(),
request.Content.Headers.ContentType.ToString()));
}
else
{
request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
accountName,
accountKey,
method,
dateTime,
request));
}
return request;
};
return requestFunc;
}
public static string GetRootRestUrl(string accountName)
{
return $"https://{accountName}.blob.core.windows.net";
}
public static string GetContainerRestUrl(string accountName, string containerName)
{
return $"{GetRootRestUrl(accountName)}/{containerName}";
}
public static string GetBlobRestUrl(string accountName, string containerName, string blob)
{
return $"{GetContainerRestUrl(accountName, containerName)}/{blob}";
}
}
}
src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/CopyAzureBlobToBlob.cs 已删除 100644 → 0
浏览文件 @ f23f0167
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
using Microsoft.Build.Framework;
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Threading.Tasks;
namespace Microsoft.DotNet.Build.CloudTestTasks
{
public partial class CopyAzureBlobToBlob : AzureConnectionStringBuildTask
{
[Required]
public string ContainerName { get; set; }
[Required]
public string SourceBlobName { get; set; }
[Required]
public string DestinationBlobName { get; set; }
public override bool Execute()
{
return ExecuteAsync().GetAwaiter().GetResult();
}
public async Task<bool> ExecuteAsync()
{
ParseConnectionString();
if (Log.HasLoggedErrors)
{
return false;
}
string sourceUrl = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, SourceBlobName);
string destinationUrl = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, DestinationBlobName);
using (HttpClient client = new HttpClient())
{
try
{
Tuple<string, string> leaseAction = new Tuple<string, string>("x-ms-lease-action", "acquire");
Tuple<string, string> leaseDuration = new Tuple<string, string>("x-ms-lease-duration", "60" /* seconds */);
Tuple<string, string> headerSource = new Tuple<string, string>("x-ms-copy-source", sourceUrl);
List<Tuple<string, string>> additionalHeaders = new List<Tuple<string, string>>() { leaseAction, leaseDuration, headerSource };
var request = AzureHelper.RequestMessage("PUT", destinationUrl, AccountName, AccountKey, additionalHeaders);
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, request))
{
if (response.IsSuccessStatusCode)
{
return true;
}
}
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
}
return false;
}
public static bool Execute(string accountName,
string accountKey,
string connectionString,
string containerName,
string sourceBlobName,
string destinationBlobName,
IBuildEngine buildengine,
ITaskHost taskHost)
{
CopyAzureBlobToBlob copyAzureBlobToBlob = new CopyAzureBlobToBlob()
{
AccountName = accountName,
AccountKey = accountKey,
ContainerName = containerName,
SourceBlobName = sourceBlobName,
DestinationBlobName = destinationBlobName,
BuildEngine = buildengine,
HostObject = taskHost
};
return copyAzureBlobToBlob.Execute();
}
public static Task<bool> ExecuteAsync(string accountName,
string accountKey,
string connectionString,
string containerName,
string sourceBlobName,
string destinationBlobName,
IBuildEngine buildengine,
ITaskHost taskHost)
{
CopyAzureBlobToBlob copyAzureBlobToBlob = new CopyAzureBlobToBlob()
{
AccountName = accountName,
AccountKey = accountKey,
ContainerName = containerName,
SourceBlobName = sourceBlobName,
DestinationBlobName = destinationBlobName,
BuildEngine = buildengine,
HostObject = taskHost
};
return copyAzureBlobToBlob.ExecuteAsync();
}
}
}
src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/DeleteAzureBlob.cs 已删除 100644 → 0
浏览文件 @ f23f0167
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
using Microsoft.Build.Framework;
using System;
using System.Collections.Generic;
using System.Net.Http;
namespace Microsoft.DotNet.Build.CloudTestTasks
{
public partial class DeleteAzureBlob: AzureConnectionStringBuildTask
{
[Required]
public string ContainerName { get; set; }
[Required]
public string BlobName { get; set; }
public override bool Execute()
{
ParseConnectionString();
if (Log.HasLoggedErrors)
{
return false;
}
string deleteUrl = $"https://{AccountName}.blob.core.windows.net/{ContainerName}/{BlobName}";
using (HttpClient client = new HttpClient())
{
try
{
Tuple<string, string> snapshots = new Tuple<string, string>("x-ms-lease-delete-snapshots", "include");
List<Tuple<string, string>> additionalHeaders = new List<Tuple<string, string>>() { snapshots };
var request = AzureHelper.RequestMessage("DELETE", deleteUrl, AccountName, AccountKey, additionalHeaders);
using (HttpResponseMessage response = AzureHelper.RequestWithRetry(Log, client, request).GetAwaiter().GetResult())
{
return response.IsSuccessStatusCode;
}
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
}
return !Log.HasLoggedErrors;
}
public static bool Execute(string accountName,
string accountKey,
string connectionString,
string containerName,
string blobName,
IBuildEngine buildengine,
ITaskHost taskHost)
{
DeleteAzureBlob deleteAzureoBlob = new DeleteAzureBlob()
{
AccountName = accountName,
AccountKey = accountKey,
ContainerName = containerName,
BlobName = blobName,
BuildEngine = buildengine,
HostObject = taskHost
};
return deleteAzureoBlob.Execute();
}
}
}
src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/DownloadFromAzure.cs 已删除 100644 → 0
浏览文件 @ f23f0167
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
using Microsoft.Build.Framework;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
namespace Microsoft.DotNet.Build.CloudTestTasks
{
public sealed class DownloadFromAzure : AzureConnectionStringBuildTask
{
/// <summary>
/// The name of the container to access. The specified name must be in the correct format, see the
/// following page for more info. https://msdn.microsoft.com/en-us/library/azure/dd135715.aspx
/// </summary>
[Required]
public string ContainerName { get; set; }
/// <summary>
/// Directory to download blob files to.
/// </summary>
[Required]
public string DownloadDirectory { get; set; }
public string BlobNamePrefix { get; set; }
public string BlobNameExtension { get; set; }
public ITaskItem[] BlobNames { get; set; }
public bool DownloadFlatFiles { get; set; }
public int MaxClients { get; set; } = 8;
private static readonly CancellationTokenSource TokenSource = new CancellationTokenSource();
private static readonly CancellationToken CancellationToken = TokenSource.Token;
public void Cancel()
{
TokenSource.Cancel();
}
public override bool Execute()
{
return ExecuteAsync(CancellationToken).GetAwaiter().GetResult();
}
public async Task<bool> ExecuteAsync(CancellationToken ct)
{
ParseConnectionString();
// If the connection string AND AccountKey & AccountName are provided, error out.
if (Log.HasLoggedErrors)
{
return false;
}
Log.LogMessage(MessageImportance.Normal, "Downloading contents of container {0} from storage account '{1}' to directory {2}.",
ContainerName, AccountName, DownloadDirectory);
try
{
List<string> blobNames = new List<string>();
if (BlobNames == null)
{
ListAzureBlobs listAzureBlobs = new ListAzureBlobs()
{
AccountName = AccountName,
AccountKey = AccountKey,
ContainerName = ContainerName,
FilterBlobNames = BlobNamePrefix,
BuildEngine = this.BuildEngine,
HostObject = this.HostObject
};
listAzureBlobs.Execute();
blobNames = listAzureBlobs.BlobNames.ToList();
}
else
{
blobNames = BlobNames.Select(b => b.ItemSpec).ToList<string>();
if (BlobNamePrefix != null)
{
blobNames = blobNames.Where(b => b.StartsWith(BlobNamePrefix)).ToList<string>();
}
}
if (BlobNameExtension != null)
{
blobNames = blobNames.Where(b => Path.GetExtension(b) == BlobNameExtension).ToList<string>();
}
if (!Directory.Exists(DownloadDirectory))
{
Directory.CreateDirectory(DownloadDirectory);
}
using (var clientThrottle = new SemaphoreSlim(this.MaxClients, this.MaxClients))
using (HttpClient client = new HttpClient())
{
client.Timeout = TimeSpan.FromMinutes(10);
await Task.WhenAll(blobNames.Select(item => DownloadItem(client, ct, item, clientThrottle)));
}
}
catch (Exception e)
{
Log.LogError(e.ToString());
}
return !Log.HasLoggedErrors;
}
private async Task DownloadItem(HttpClient client, CancellationToken ct, string blob, SemaphoreSlim clientThrottle)
{
await clientThrottle.WaitAsync();
string filename = string.Empty;
try {
Log.LogMessage(MessageImportance.Low, "Downloading BLOB - {0}", blob);
string blobUrl = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, blob);
filename = Path.Combine(DownloadDirectory, Path.GetFileName(blob));
if (!DownloadFlatFiles)
{
int dirIndex = blob.LastIndexOf("/");
string blobDirectory = string.Empty;
string blobFilename = string.Empty;
if (dirIndex == -1)
{
blobFilename = blob;
}
else
{
blobDirectory = blob.Substring(0, dirIndex);
blobFilename = blob.Substring(dirIndex + 1);
// Trim blob name prefix (directory part) from download to blob directory
if (BlobNamePrefix != null)
{
if (BlobNamePrefix.Length > dirIndex)
{
BlobNamePrefix = BlobNamePrefix.Substring(0, dirIndex);
}
blobDirectory = blobDirectory.Substring(BlobNamePrefix.Length);
}
}
string downloadBlobDirectory = Path.Combine(DownloadDirectory, blobDirectory);
if (!Directory.Exists(downloadBlobDirectory))
{
Directory.CreateDirectory(downloadBlobDirectory);
}
filename = Path.Combine(downloadBlobDirectory, blobFilename);
}
var createRequest = AzureHelper.RequestMessage("GET", blobUrl, AccountName, AccountKey);
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
if (response.IsSuccessStatusCode)
{
// Blobs can be files but have the name of a directory. We'll skip those and log something weird happened.
if (!string.IsNullOrEmpty(Path.GetFileName(filename)))
{
Stream responseStream = await response.Content.ReadAsStreamAsync();
using (FileStream sourceStream = File.Open(filename, FileMode.Create))
{
responseStream.CopyTo(sourceStream);
}
}
else
{
Log.LogWarning($"Unable to download blob '{blob}' as it has a directory-like name. This may cause problems if it was needed.");
}
}
else
{
Log.LogError("Failed to retrieve blob {0}, the status code was {1}", blob, response.StatusCode);
}
}
}
catch (PathTooLongException)
{
Log.LogError($"Unable to download blob as it exceeds the maximum allowed path length. Path: {filename}. Length:{filename.Length}");
}
catch (Exception ex)
{
Log.LogError(ex.ToString());
}
finally
{
clientThrottle.Release();
}
}
}
}
src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/ListAzureBlobs.cs 已删除 100644 → 0
浏览文件 @ f23f0167
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using System.Xml;
namespace Microsoft.DotNet.Build.CloudTestTasks
{
public partial class ListAzureBlobs : AzureConnectionStringBuildTask
{
/// <summary>
/// The name of the container to access. The specified name must be in the correct format, see the
/// following page for more info. https://msdn.microsoft.com/en-us/library/azure/dd135715.aspx
/// </summary>
[Required]
public string ContainerName { get; set; }
[Output]
public string[] BlobNames { get; set; }
public string FilterBlobNames { get; set; }
public override bool Execute()
{
return ExecuteAsync().GetAwaiter().GetResult();
}
public static string[] Execute(string accountName,
string accountKey,
string connectionString,
string containerName,
string filterBlobNames,
IBuildEngine buildengine,
ITaskHost taskHost)
{
ListAzureBlobs getAzureBlobList = new ListAzureBlobs()
{
AccountName = accountName,
AccountKey = accountKey,
ContainerName = containerName,
FilterBlobNames = filterBlobNames,
BuildEngine = buildengine,
HostObject = taskHost
};
getAzureBlobList.Execute();
return getAzureBlobList.BlobNames;
}
// This code is duplicated in BuildTools task DownloadFromAzure, and that code should be refactored to permit blob listing.
public async Task<bool> ExecuteAsync()
{
ParseConnectionString();
try
{
List<string> blobNames = await ListBlobs(Log, AccountName, AccountKey, ContainerName, FilterBlobNames);
BlobNames = blobNames.ToArray();
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
return !Log.HasLoggedErrors;
}
public static async Task<List<string>> ListBlobs(TaskLoggingHelper Log, string AccountName, string AccountKey, string ContainerName, string FilterBlobNames)
{
List<string> blobsNames = new List<string>();
string urlListBlobs = string.Format("https://{0}.blob.core.windows.net/{1}?restype=container&comp=list", AccountName, ContainerName);
if (!string.IsNullOrWhiteSpace(FilterBlobNames))
{
urlListBlobs += $"&prefix={FilterBlobNames}";
}
Log.LogMessage(MessageImportance.Low, "Sending request to list blobsNames for container '{0}'.", ContainerName);
using (HttpClient client = new HttpClient())
{
var createRequest = AzureHelper.RequestMessage("GET", urlListBlobs, AccountName, AccountKey);
XmlDocument responseFile;
string nextMarker = string.Empty;
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
responseFile = new XmlDocument();
responseFile.LoadXml(await response.Content.ReadAsStringAsync());
XmlNodeList elemList = responseFile.GetElementsByTagName("Name");
blobsNames.AddRange(elemList.Cast<XmlNode>()
.Select(x => x.InnerText)
.ToList());
nextMarker = responseFile.GetElementsByTagName("NextMarker").Cast<XmlNode>().FirstOrDefault()?.InnerText;
}
while (!string.IsNullOrEmpty(nextMarker))
{
urlListBlobs = $"https://{AccountName}.blob.core.windows.net/{ContainerName}?restype=container&comp=list&marker={nextMarker}";
if (!string.IsNullOrWhiteSpace(FilterBlobNames))
{
urlListBlobs += $"&prefix={FilterBlobNames}";
}
var nextRequest = AzureHelper.RequestMessage("GET", urlListBlobs, AccountName, AccountKey);
using (HttpResponseMessage nextResponse = AzureHelper.RequestWithRetry(Log, client, nextRequest).GetAwaiter().GetResult())
{
responseFile = new XmlDocument();
responseFile.LoadXml(await nextResponse.Content.ReadAsStringAsync());
XmlNodeList elemList = responseFile.GetElementsByTagName("Name");
blobsNames.AddRange(elemList.Cast<XmlNode>()
.Select(x => x.InnerText)
.ToList());
nextMarker = responseFile.GetElementsByTagName("NextMarker").Cast<XmlNode>().FirstOrDefault()?.InnerText;
}
}
}
return blobsNames;
}
}
}
src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/PublishStringToAzureBlob.cs 已删除 100644 → 0
浏览文件 @ f23f0167
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
using Microsoft.Build.Framework;
using System;
using System.Collections.Generic;
using System.Net.Http;
namespace Microsoft.DotNet.Build.CloudTestTasks
{
public partial class PublishStringToAzureBlob : AzureConnectionStringBuildTask
{
[Required]
public string BlobName { get; set; }
[Required]
public string ContainerName { get; set; }
[Required]
public string Content { get; set; }
public string ContentType { get; set; }
public override bool Execute()
{
ParseConnectionString();
string blobUrl = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, BlobName);
using (HttpClient client = new HttpClient())
{
try
{
Tuple<string, string> headerBlobType = new Tuple<string, string>("x-ms-blob-type", "BlockBlob");
List<Tuple<string, string>> additionalHeaders = new List<Tuple<string, string>>() { headerBlobType };
if (!string.IsNullOrEmpty(ContentType))
{
additionalHeaders.Add(new Tuple<string, string>(AzureHelper.ContentTypeString, ContentType));
}
var request = AzureHelper.RequestMessage("PUT", blobUrl, AccountName, AccountKey, additionalHeaders, Content);
AzureHelper.RequestWithRetry(Log, client, request).GetAwaiter().GetResult();
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
}
return !Log.HasLoggedErrors;
}
public static bool Execute(string accountName,
string accountKey,
string connectionString,
string containerName,
string blobName,
string content,
string contentType,
IBuildEngine buildengine,
ITaskHost taskHost)
{
PublishStringToAzureBlob publishStringToBlob = new PublishStringToAzureBlob()
{
AccountName = accountName,
AccountKey = accountKey,
ContainerName = containerName,
BlobName = blobName,
Content = content,
ContentType = contentType,
BuildEngine = buildengine,
HostObject = taskHost
};
return publishStringToBlob.Execute();
}
}
}
src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/UploadClient.cs 已删除 100644 → 0
浏览文件 @ f23f0167
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Task = System.Threading.Tasks.Task;
namespace Microsoft.DotNet.Build.CloudTestTasks
{
public class UploadClient
{
private TaskLoggingHelper log;
public UploadClient(TaskLoggingHelper loggingHelper)
{
log = loggingHelper;
}
public string EncodeBlockIds(int numberOfBlocks, int lengthOfId)
{
string numberOfBlocksString = numberOfBlocks.ToString("D" + lengthOfId);
if (Encoding.UTF8.GetByteCount(numberOfBlocksString) <= 64)
{
byte[] bytes = Encoding.UTF8.GetBytes(numberOfBlocksString);
return Convert.ToBase64String(bytes);
}
else
{
throw new Exception("Task failed - Could not encode block id.");
}
}
public async Task UploadBlockBlobAsync(
CancellationToken ct,
string AccountName,
string AccountKey,
string ContainerName,
string filePath,
string destinationBlob,
string contentType,
int uploadTimeout,
string leaseId = "")
{
string resourceUrl = AzureHelper.GetContainerRestUrl(AccountName, ContainerName);
string fileName = destinationBlob;
fileName = fileName.Replace("\\", "/");
string blobUploadUrl = resourceUrl + "/" + fileName;
int size = (int)new FileInfo(filePath).Length;
int blockSize = 4 * 1024 * 1024; //4MB max size of a block blob
int bytesLeft = size;
List<string> blockIds = new List<string>();
int numberOfBlocks = (size / blockSize) + 1;
int countForId = 0;
using (FileStream fileStreamTofilePath = new FileStream(filePath, FileMode.Open, FileAccess.Read))
{
int offset = 0;
while (bytesLeft > 0)
{
int nextBytesToRead = (bytesLeft < blockSize) ? bytesLeft : blockSize;
byte[] fileBytes = new byte[blockSize];
int read = fileStreamTofilePath.Read(fileBytes, 0, nextBytesToRead);
if (nextBytesToRead != read)
{
throw new Exception(string.Format(
"Number of bytes read ({0}) from file {1} isn't equal to the number of bytes expected ({2}) .",
read, fileName, nextBytesToRead));
}
string blockId = EncodeBlockIds(countForId, numberOfBlocks.ToString().Length);
blockIds.Add(blockId);
string blockUploadUrl = blobUploadUrl + "?comp=block&blockid=" + WebUtility.UrlEncode(blockId);
using (HttpClient client = new HttpClient())
{
client.DefaultRequestHeaders.Clear();
// In random occassions the request fails if the network is slow and it takes more than 100 seconds to upload 4MB.
client.Timeout = TimeSpan.FromMinutes(uploadTimeout);
Func<HttpRequestMessage> createRequest = () =>
{
DateTime dt = DateTime.UtcNow;
var req = new HttpRequestMessage(HttpMethod.Put, blockUploadUrl);
req.Headers.Add(
AzureHelper.DateHeaderString,
dt.ToString("R", CultureInfo.InvariantCulture));
req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
if (!string.IsNullOrWhiteSpace(leaseId))
{
log.LogMessage($"Sending request: {leaseId} {blockUploadUrl}");
req.Headers.Add("x-ms-lease-id", leaseId);
}
req.Headers.Add(
AzureHelper.AuthorizationHeaderString,
AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"PUT",
dt,
req,
string.Empty,
string.Empty,
nextBytesToRead.ToString(),
string.Empty));
Stream postStream = new MemoryStream();
postStream.Write(fileBytes, 0, nextBytesToRead);
postStream.Seek(0, SeekOrigin.Begin);
req.Content = new StreamContent(postStream);
return req;
};
log.LogMessage(MessageImportance.Low, "Sending request to upload part {0} of file {1}", countForId, fileName);
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(log, client, createRequest))
{
log.LogMessage(
MessageImportance.Low,
"Received response to upload part {0} of file {1}: Status Code:{2} Status Desc: {3}",
countForId,
fileName,
response.StatusCode,
await response.Content.ReadAsStringAsync());
}
}
offset += read;
bytesLeft -= nextBytesToRead;
countForId += 1;
}
}
string blockListUploadUrl = blobUploadUrl + "?comp=blocklist";
using (HttpClient client = new HttpClient())
{
Func<HttpRequestMessage> createRequest = () =>
{
DateTime dt1 = DateTime.UtcNow;
var req = new HttpRequestMessage(HttpMethod.Put, blockListUploadUrl);
req.Headers.Add(AzureHelper.DateHeaderString, dt1.ToString("R", CultureInfo.InvariantCulture));
req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
if (string.IsNullOrEmpty(contentType))
{
contentType = DetermineContentTypeBasedOnFileExtension(filePath);
}
if (!string.IsNullOrEmpty(contentType))
{
req.Headers.Add(AzureHelper.ContentTypeString, contentType);
}
string cacheControl = DetermineCacheControlBasedOnFileExtension(filePath);
if (!string.IsNullOrEmpty(cacheControl))
{
req.Headers.Add(AzureHelper.CacheControlString, cacheControl);
}
var body = new StringBuilder("<?xml version=\"1.0\" encoding=\"UTF-8\"?><BlockList>");
foreach (object item in blockIds)
body.AppendFormat("<Latest>{0}</Latest>", item);
body.Append("</BlockList>");
byte[] bodyData = Encoding.UTF8.GetBytes(body.ToString());
if (!string.IsNullOrWhiteSpace(leaseId))
{
log.LogMessage($"Sending list request: {leaseId} {blockListUploadUrl}");
req.Headers.Add("x-ms-lease-id", leaseId);
}
req.Headers.Add(
AzureHelper.AuthorizationHeaderString,
AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"PUT",
dt1,
req,
string.Empty,
string.Empty,
bodyData.Length.ToString(),
string.Empty));
Stream postStream = new MemoryStream();
postStream.Write(bodyData, 0, bodyData.Length);
postStream.Seek(0, SeekOrigin.Begin);
req.Content = new StreamContent(postStream);
return req;
};
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(log, client, createRequest))
{
log.LogMessage(
MessageImportance.Low,
"Received response to combine block list for file {0}: Status Code:{1} Status Desc: {2}",
fileName,
response.StatusCode,
await response.Content.ReadAsStringAsync());
}
}
}
public async Task<bool> FileEqualsExistingBlobAsync(
string accountName,
string accountKey,
string containerName,
string filePath,
string destinationBlob,
int uploadTimeout)
{
using (var client = new HttpClient
{
Timeout = TimeSpan.FromMinutes(uploadTimeout)
})
{
log.LogMessage(
MessageImportance.Low,
$"Downloading blob {destinationBlob} to check if identical.");
string blobUrl = AzureHelper.GetBlobRestUrl(accountName, containerName, destinationBlob);
var createRequest = AzureHelper.RequestMessage("GET", blobUrl, accountName, accountKey);
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(
log,
client,
createRequest))
{
if (!response.IsSuccessStatusCode)
{
throw new HttpRequestException(
$"Failed to retrieve existing blob {destinationBlob}, " +
$"status code {response.StatusCode}.");
}
byte[] existingBytes = await response.Content.ReadAsByteArrayAsync();
byte[] localBytes = File.ReadAllBytes(filePath);
bool equal = localBytes.SequenceEqual(existingBytes);
if (equal)
{
log.LogMessage(
MessageImportance.Normal,
"Item exists in blob storage, and is verified to be identical. " +
$"File: '{filePath}' Blob: '{destinationBlob}'");
}
return equal;
}
}
}
private string DetermineContentTypeBasedOnFileExtension(string filename)
{
if (Path.GetExtension(filename) == ".svg")
{
return "image/svg+xml";
}
else if (Path.GetExtension(filename) == ".version")
{
return "text/plain";
}
return string.Empty;
}
private string DetermineCacheControlBasedOnFileExtension(string filename)
{
if (Path.GetExtension(filename) == ".svg")
{
return "No-Cache";
}
return string.Empty;
}
}
}
src/tasks/installer.tasks/BuildTools.Publish/CloudTestTasks/UploadToAzure.cs 已删除 100644 → 0
浏览文件 @ f23f0167
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
using Microsoft.Build.Framework;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using System.Xml;
using ThreadingTask = System.Threading.Tasks.Task;
namespace Microsoft.DotNet.Build.CloudTestTasks
{
public class UploadToAzure : AzureConnectionStringBuildTask, ICancelableTask
{
private static readonly CancellationTokenSource TokenSource = new CancellationTokenSource();
private static readonly CancellationToken CancellationToken = TokenSource.Token;
/// <summary>
/// The name of the container to access. The specified name must be in the correct format, see the
/// following page for more info. https://msdn.microsoft.com/en-us/library/azure/dd135715.aspx
/// </summary>
[Required]
public string ContainerName { get; set; }
/// <summary>
/// An item group of files to upload. Each item must have metadata RelativeBlobPath
/// that specifies the path relative to ContainerName where the item will be uploaded.
/// </summary>
[Required]
public ITaskItem[] Items { get; set; }
/// <summary>
/// Indicates if the destination blob should be overwritten if it already exists. The default if false.
/// </summary>
public bool Overwrite { get; set; } = false;
/// <summary>
/// Enables idempotency when Overwrite is false.
///
/// false: (default) Attempting to upload an item that already exists fails.
///
/// true: When an item already exists, download the existing blob to check if it's
/// byte-for-byte identical to the one being uploaded. If so, pass. If not, fail.
/// </summary>
public bool PassIfExistingItemIdentical { get; set; }
/// <summary>
/// Specifies the maximum number of clients to concurrently upload blobs to azure
/// </summary>
public int MaxClients { get; set; } = 8;
public int UploadTimeoutInMinutes { get; set; } = 5;
public void Cancel()
{
TokenSource.Cancel();
}
public override bool Execute()
{
return ExecuteAsync(CancellationToken).GetAwaiter().GetResult();
}
public async Task<bool> ExecuteAsync(CancellationToken ct)
{
ParseConnectionString();
// If the connection string AND AccountKey & AccountName are provided, error out.
if (Log.HasLoggedErrors)
{
return false;
}
Log.LogMessage(
MessageImportance.Normal,
"Begin uploading blobs to Azure account {0} in container {1}.",
AccountName,
ContainerName);
if (Items.Length == 0)
{
Log.LogError("No items were provided for upload.");
return false;
}
// first check what blobs are present
string checkListUrl = $"{AzureHelper.GetContainerRestUrl(AccountName, ContainerName)}?restype=container&comp=list";
HashSet<string> blobsPresent = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
try
{
using (HttpClient client = new HttpClient())
{
var createRequest = AzureHelper.RequestMessage("GET", checkListUrl, AccountName, AccountKey);
Log.LogMessage(MessageImportance.Low, "Sending request to check whether Container blobs exist");
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
var doc = new XmlDocument();
doc.LoadXml(await response.Content.ReadAsStringAsync());
XmlNodeList nodes = doc.DocumentElement.GetElementsByTagName("Blob");
foreach (XmlNode node in nodes)
{
blobsPresent.Add(node["Name"].InnerText);
}
Log.LogMessage(MessageImportance.Low, "Received response to check whether Container blobs exist");
}
}
using (var clientThrottle = new SemaphoreSlim(this.MaxClients, this.MaxClients))
{
await ThreadingTask.WhenAll(Items.Select(item => UploadAsync(ct, item, blobsPresent, clientThrottle)));
}
Log.LogMessage(MessageImportance.Normal, "Upload to Azure is complete, a total of {0} items were uploaded.", Items.Length);
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
return !Log.HasLoggedErrors;
}
private async ThreadingTask UploadAsync(CancellationToken ct, ITaskItem item, HashSet<string> blobsPresent, SemaphoreSlim clientThrottle)
{
if (ct.IsCancellationRequested)
{
Log.LogError("Task UploadToAzure cancelled");
ct.ThrowIfCancellationRequested();
}
string relativeBlobPath = item.GetMetadata("RelativeBlobPath");
if (string.IsNullOrEmpty(relativeBlobPath))
throw new Exception(string.Format("Metadata 'RelativeBlobPath' is missing for item '{0}'.", item.ItemSpec));
if (!File.Exists(item.ItemSpec))
throw new Exception(string.Format("The file '{0}' does not exist.", item.ItemSpec));
UploadClient uploadClient = new UploadClient(Log);
if (!Overwrite && blobsPresent.Contains(relativeBlobPath))
{
if (PassIfExistingItemIdentical &&
await ItemEqualsExistingBlobAsync(item, relativeBlobPath, uploadClient, clientThrottle))
{
return;
}
throw new Exception(string.Format("The blob '{0}' already exists.", relativeBlobPath));
}
string contentType = item.GetMetadata("ContentType");
await clientThrottle.WaitAsync();
try
{
Log.LogMessage("Uploading {0} to {1}.", item.ItemSpec, ContainerName);
await
uploadClient.UploadBlockBlobAsync(
ct,
AccountName,
AccountKey,
ContainerName,
item.ItemSpec,
relativeBlobPath,
contentType,
UploadTimeoutInMinutes);
}
finally
{
clientThrottle.Release();
}
}
private async Task<bool> ItemEqualsExistingBlobAsync(
ITaskItem item,
string relativeBlobPath,
UploadClient client,
SemaphoreSlim clientThrottle)
{
await clientThrottle.WaitAsync();
try
{
return await client.FileEqualsExistingBlobAsync(
AccountName,
AccountKey,
ContainerName,
item.ItemSpec,
relativeBlobPath,
UploadTimeoutInMinutes);
}
finally
{
clientThrottle.Release();
}
}
}
}
src/tasks/installer.tasks/BuildTools.Publish/README.md 已删除 100644 → 0
浏览文件 @ f23f0167
The files in this directory are the closure of C# code from the BuildTools repo
that's necessary for the Core-Setup publish tasks. There are no changes except
automatically removing and sorting the using statements.
Source: https://github.com/dotnet/buildtools/tree/55d43483866c7caeeace96355add3a9b12fa5795
Using the existing BuildTools code reduces the risk of behavior differences vs.
trying to find equivalents in Arcade. The upcoming new Arcade-powered publish
functionality makes short-term effort to deduplicate these tasks throwaway work.
See [core-setup/#8285 "Migrate to Arcade's blob publish infrastructure"](https://github.com/dotnet/core-setup/issues/8285)
src/tasks/installer.tasks/FinalizeBuild.cs 已删除 100644 → 0
浏览文件 @ f23f0167
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
using Microsoft.Build.Framework;
using Microsoft.DotNet.Build.CloudTestTasks;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
namespace Microsoft.DotNet.Build.Tasks
{
public class FinalizeBuild : AzureConnectionStringBuildTask
{
[Required]
public string SemaphoreBlob { get; set; }
[Required]
public string FinalizeContainer { get; set; }
public string MaxWait { get; set; }
public string Delay { get; set; }
[Required]
public string ContainerName { get; set; }
[Required]
public string Channel { get; set; }
[Required]
public string SharedFrameworkNugetVersion { get; set; }
[Required]
public string SharedHostNugetVersion { get; set; }
[Required]
public string ProductVersion { get; set; }
[Required]
public string Version { get; set; }
[Required]
public string CommitHash { get; set; }
public bool ForcePublish { get; set; }
private Regex _versionRegex = new Regex(@"(?<version>\d+\.\d+\.\d+)(-(?<prerelease>[^-]+-)?(?<major>\d+)-(?<minor>\d+))?");
public override bool Execute()
{
ParseConnectionString();
if (Log.HasLoggedErrors)
{
return false;
}
if (!FinalizeContainer.EndsWith("/"))
{
FinalizeContainer = $"{FinalizeContainer}/";
}
string targetVersionFile = $"{FinalizeContainer}{Version}";
CreateBlobIfNotExists(SemaphoreBlob);
AzureBlobLease blobLease = new AzureBlobLease(AccountName, AccountKey, ConnectionString, ContainerName, SemaphoreBlob, Log);
Log.LogMessage($"Acquiring lease on semaphore blob '{SemaphoreBlob}'");
blobLease.Acquire();
// Prevent race conditions by dropping a version hint of what version this is. If we see this file
// and it is the same as our version then we know that a race happened where two+ builds finished
// at the same time and someone already took care of publishing and we have no work to do.
if (IsLatestSpecifiedVersion(targetVersionFile) && !ForcePublish)
{
Log.LogMessage(MessageImportance.Low, $"version hint file for publishing finalization is {targetVersionFile}");
Log.LogMessage(MessageImportance.High, $"Version '{Version}' is already published, skipping finalization.");
Log.LogMessage($"Releasing lease on semaphore blob '{SemaphoreBlob}'");
blobLease.Release();
return true;
}
else
{
// Delete old version files
GetBlobList(FinalizeContainer)
.Select(s => s.Replace("/dotnet/", ""))
.Where(w => _versionRegex.Replace(Path.GetFileName(w), "") == "")
.ToList()
.ForEach(f => TryDeleteBlob(f));
// Drop the version file signaling such for any race-condition builds (see above comment).
CreateBlobIfNotExists(targetVersionFile);
try
{
CopyBlobs($"Runtime/{ProductVersion}/", $"Runtime/{Channel}/");
// Generate the latest version text file
string sfxVersion = GetSharedFrameworkVersionFileContent();
PublishStringToBlob(ContainerName, $"Runtime/{Channel}/latest.version", sfxVersion, "text/plain");
}
finally
{
blobLease.Release();
}
}
return !Log.HasLoggedErrors;
}
private string GetSharedFrameworkVersionFileContent()
{
string returnString = $"{CommitHash}{Environment.NewLine}";
returnString += $"{SharedFrameworkNugetVersion}{Environment.NewLine}";
return returnString;
}
public bool CopyBlobs(string sourceFolder, string destinationFolder)
{
bool returnStatus = true;
List<Task<bool>> copyTasks = new List<Task<bool>>();
string[] blobs = GetBlobList(sourceFolder);
foreach (string blob in blobs)
{
string targetName = Path.GetFileName(blob)
.Replace(SharedFrameworkNugetVersion, "latest")
.Replace(SharedHostNugetVersion, "latest");
string sourceBlob = blob.Replace($"/{ContainerName}/", "");
string destinationBlob = $"{destinationFolder}{targetName}";
Log.LogMessage($"Copying blob '{sourceBlob}' to '{destinationBlob}'");
copyTasks.Add(CopyBlobAsync(sourceBlob, destinationBlob));
}
Task.WaitAll(copyTasks.ToArray());
copyTasks.ForEach(c => returnStatus &= c.Result);
return returnStatus;
}
public bool TryDeleteBlob(string path)
{
return DeleteBlob(ContainerName, path);
}
public void CreateBlobIfNotExists(string path)
{
var blobList = GetBlobList(path);
if(blobList.Count() == 0)
{
PublishStringToBlob(ContainerName, path, DateTime.Now.ToString());
}
}
public bool IsLatestSpecifiedVersion(string versionFile)
{
var blobList = GetBlobList(versionFile);
return blobList.Count() != 0;
}
public bool DeleteBlob(string container, string blob)
{
return DeleteAzureBlob.Execute(AccountName,
AccountKey,
ConnectionString,
container,
blob,
BuildEngine,
HostObject);
}
public Task<bool> CopyBlobAsync(string sourceBlobName, string destinationBlobName)
{
return CopyAzureBlobToBlob.ExecuteAsync(AccountName,
AccountKey,
ConnectionString,
ContainerName,
sourceBlobName,
destinationBlobName,
BuildEngine,
HostObject);
}
public string[] GetBlobList(string path)
{
return ListAzureBlobs.Execute(AccountName,
AccountKey,
ConnectionString,
ContainerName,
path,
BuildEngine,
HostObject);
}
public bool PublishStringToBlob(string container, string blob, string contents, string contentType = null)
{
return PublishStringToAzureBlob.Execute(AccountName,
AccountKey,
ConnectionString,
container,
blob,
contents,
contentType,
BuildEngine,
HostObject);
}
}
}
src/tasks/installer.tasks/installer.tasks.csproj
浏览文件 @ cdff28e8
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFrameworks>netstandard2.0;net461</TargetFrameworks>
<EnableDefaultCompileItems>false</EnableDefaultCompileItems>
<TargetFrameworks>$(NetCoreAppToolCurrent);net461</TargetFrameworks>
<IsShipping>false</IsShipping>
<RunAnalyzers>false</RunAnalyzers>
</PropertyGroup>
<ItemGroup>
<Compile Include="*.cs" />
<Compile Include="BuildTools.Publish\**\*.cs" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.DependencyModel" Version="$(MicrosoftExtensionsDependencyModelVersion)" />
<PackageReference Include="NuGet.ProjectModel" Version="$(RefOnlyNugetProjectModelVersion)" />
<PackageReference Include="NuGet.Packaging" Version="$(RefOnlyNugetPackagingVersion)" />
<PackageReference Include="System.Reflection.Metadata" Version="1.7.0" />
</ItemGroup>
<ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">
<ItemGroup Condition="'$(TargetFramework)' == '$(NetCoreAppToolCurrent)'">
<PackageReference Include="Microsoft.Build" Version="$(RefOnlyMicrosoftBuildVersion)" />
<PackageReference Include="Microsoft.Build.Framework" Version="$(RefOnlyMicrosoftBuildFrameworkVersion)" />
<PackageReference Include="Microsoft.Build.Tasks.Core" Version="$(RefOnlyMicrosoftBuildTasksCoreVersion)" />
<PackageReference Include="Microsoft.Build.Utilities.Core" Version="$(RefOnlyMicrosoftBuildUtilitiesCoreVersion)" />
<PackageReference Include="System.Diagnostics.FileVersionInfo" Version="4.0.0" />
</ItemGroup>
<ItemGroup Condition="'$(TargetFramework)' == 'net461'">
<PackageReference Include="System.Reflection.Metadata" Version="$(SystemReflectionMetadataVersion)" />
<Reference Include="Microsoft.Build.Framework" />
<Reference Include="Microsoft.Build.Tasks.v4.0" />
<Reference Include="Microsoft.Build.Utilities.v4.0" />
<Reference Include="Microsoft.Build" />
<Reference Include="System.IO.Compression.FileSystem" />
<Reference Include="System.IO.Compression" />
</ItemGroup>
</Project>
src/tasks/tasks.proj
浏览文件 @ cdff28e8
......@@ -19,7 +19,7 @@
locking issues vs. the persistent nodes that loaded the task DLL for the previous build. It
isn't particularly accurate, but better than nothing.
-->
<Target Name="BuildAndRestoreIncrementally"
<Target Name="BuildIncrementally"
DependsOnTargets="GetTasksSrc"
Inputs="@(TasksSrc)"
Outputs="$(TasksIntermediateFile)">
......@@ -27,10 +27,6 @@
<TaskProject Include="$(MSBuildProjectFullPath)" />
</ItemGroup>
<MSBuild Projects="@(TaskProject)"
Properties="MSBuildRestoreSessionId=$([System.Guid]::NewGuid());Configuration=Debug;Platform=AnyCPU;RESTORE_TASK_BINLOG_PARAMETERS=$(ArtifactsLogDir)Restore-tasks.proj.binlog"
Targets="Restore" />
<MSBuild Projects="@(TaskProject)"
Properties="Configuration=Debug;Platform=AnyCPU"
Targets="Build" />
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或