retry all cosmosdb ops

build/azure-pipelines/common/createAsset.ts

@@ -11,6 +11,7 @@ import * as crypto from 'crypto';
 import * as azure from 'azure-storage';
 import * as mime from 'mime';
 import { CosmosClient } from '@azure/cosmos';
+import { retry } from './retry';
 
 interface Asset {
 	platform: string;
@@ -66,26 +67,6 @@ function getEnv(name: string): string {
 	return result;
 }
 
-async function retry<T>(fn: () => Promise<T>): Promise<T> {
-	for (let run = 1; run <= 10; run++) {
-		try {
-			return await fn();
-		} catch (err) {
-			if (!/ECONNRESET/.test(err.message)) {
-				throw err;
-			}
-
-			const millis = (Math.random() * 200) + (50 * Math.pow(1.5, run));
-			console.log(`Creating build asset failed with ECONNRESET, retrying in ${millis}ms...`);
-
-			// maximum delay is 10th retry: ~3 seconds
-			await new Promise(c => setTimeout(c, millis));
-		}
-	}
-
-	throw new Error('Retried too many times');
-}
-
 async function main(): Promise<void> {
 	const [, , platform, type, fileName, filePath] = process.argv;
 	const quality = getEnv('VSCODE_QUALITY');

build/azure-pipelines/common/createBuild.ts

@@ -6,6 +6,7 @@
 'use strict';
 
 import { CosmosClient } from '@azure/cosmos';
+import { retry } from './retry';
 
 if (process.argv.length !== 3) {
 	console.error('Usage: node createBuild.js VERSION');
@@ -48,7 +49,7 @@ async function main(): Promise<void> {
 
 	const client = new CosmosClient({ endpoint: process.env['AZURE_DOCUMENTDB_ENDPOINT']!, key: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
 	const scripts = client.database('builds').container(quality).scripts;
-	await scripts.storedProcedure('createBuild').execute('', [{ ...build, _partitionKey: '' }]);
+	await retry(() => scripts.storedProcedure('createBuild').execute('', [{ ...build, _partitionKey: '' }]));
 }
 
 main().then(() => {

build/azure-pipelines/common/releaseBuild.ts

@@ -6,6 +6,7 @@
 'use strict';
 
 import { CosmosClient } from '@azure/cosmos';
+import { retry } from './retry';
 
 function getEnv(name: string): string {
 	const result = process.env[name];
@@ -58,7 +59,7 @@ async function main(): Promise<void> {
 	console.log(`Releasing build ${commit}...`);
 
 	const scripts = client.database('builds').container(quality).scripts;
-	await scripts.storedProcedure('releaseBuild').execute('', [commit]);
+	await retry(() => scripts.storedProcedure('releaseBuild').execute('', [commit]));
 }
 
 main().then(() => {

build/azure-pipelines/common/retry.ts

+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+export async function retry<T>(fn: () => Promise<T>): Promise<T> {
+	for (let run = 1; run <= 10; run++) {
+		try {
+			return await fn();
+		} catch (err) {
+			if (!/ECONNRESET/.test(err.message)) {
+				throw err;
+			}
+
+			const millis = (Math.random() * 200) + (50 * Math.pow(1.5, run));
+			console.log(`Failed with ECONNRESET, retrying in ${millis}ms...`);
+
+			// maximum delay is 10th retry: ~3 seconds
+			await new Promise(c => setTimeout(c, millis));
+		}
+	}
+
+	throw new Error('Retried too many times');
+}

build/azure-pipelines/common/sync-mooncake.ts

@@ -9,6 +9,7 @@ import * as url from 'url';
 import * as azure from 'azure-storage';
 import * as mime from 'mime';
 import { CosmosClient } from '@azure/cosmos';
+import { retry } from './retry';
 
 function log(...args: any[]) {
 	console.log(...[`[${new Date().toISOString()}]`, ...args]);
@@ -99,8 +100,8 @@ async function sync(commit: string, quality: string): Promise<void> {
 
 			log(`  Updating build in DB...`);
 			const mooncakeUrl = `${process.env['MOONCAKE_CDN_URL']}${blobPath}`;
-			await container.scripts.storedProcedure('setAssetMooncakeUrl')
-				.execute('', [commit, asset.platform, asset.type, mooncakeUrl]);
+			await retry(() => container.scripts.storedProcedure('setAssetMooncakeUrl')
+				.execute('', [commit, asset.platform, asset.type, mooncakeUrl]));
 
 			log(`  Done ✔️`);
 		} catch (err) {