download extensions at build time

build/gulpfile.vscode.js

@@ -18,6 +18,7 @@ const filter = require('gulp-filter');
 const json = require('gulp-json-editor');
 const _ = require('underscore');
 const util = require('./lib/util');
+const ext = require('./lib/extensions');
 const buildfile = require('../src/buildfile');
 const common = require('./gulpfile.common');
 const nlsDev = require('vscode-nls-dev');
@@ -35,6 +36,10 @@ const nodeModules = ['electron', 'original-fs']
 
 // Build
 
+const builtInExtensions = [
+	// { name: 'vscodevim.vim', version: '0.2.0' }
+];
+
 const vscodeEntryPoints = _.flatten([
 	buildfile.entrypoint('vs/workbench/workbench.main'),
 	buildfile.base,
@@ -167,7 +172,17 @@ function packageTask(platform, arch, opts) {
 			'!extensions/vscode-colorize-tests/**'
 		], { base: '.' });
 
-		const sources = es.merge(src, extensions)
+		const marketplaceExtensions = es.merge(...builtInExtensions.map(extension => {
+			return ext.src(extension.name, extension.version)
+				.pipe(rename(p => p.dirname = `extensions/${ extension.name }/${ p.dirname }`));
+		}));
+
+		const allExtensions = es.merge(
+			extensions,
+			marketplaceExtensions
+		);
+
+		const sources = es.merge(src, allExtensions)
 			.pipe(nlsDev.createAdditionalLanguageFiles(languages, path.join(__dirname, '..', 'i18n')))
 			.pipe(filter(['**', '!**/*.js.map']))
 			.pipe(util.handleAzureJson({ platform }));

build/lib/extensions.js

+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+"use strict";
+var event_stream_1 = require('event-stream');
+var assign = require('object-assign');
+var remote = require('gulp-remote-src');
+var flatmap = require('gulp-flatmap');
+var vzip = require('gulp-vinyl-zip');
+var filter = require('gulp-filter');
+var rename = require('gulp-rename');
+var util = require('gulp-util');
+var buffer = require('gulp-buffer');
+var json = require('gulp-json-editor');
+function error(err) {
+    var result = event_stream_1.through();
+    setTimeout(function () { return result.emit('error', err); });
+    return result;
+}
+var baseHeaders = {
+    'X-Market-Client-Id': 'VSCode Build',
+    'User-Agent': 'VSCode Build',
+};
+function src(extensionName, version) {
+    var filterType = 7;
+    var value = extensionName;
+    var criterium = { filterType: filterType, value: value };
+    var criteria = [criterium];
+    var pageNumber = 1;
+    var pageSize = 1;
+    var sortBy = 0;
+    var sortOrder = 0;
+    var flags = 0x1 | 0x2 | 0x80;
+    var assetTypes = ['Microsoft.VisualStudio.Services.VSIXPackage'];
+    var filters = [{ criteria: criteria, pageNumber: pageNumber, pageSize: pageSize, sortBy: sortBy, sortOrder: sortOrder }];
+    var body = JSON.stringify({ filters: filters, assetTypes: assetTypes, flags: flags });
+    var headers = assign({}, baseHeaders, {
+        'Content-Type': 'application/json',
+        'Accept': 'application/json;api-version=3.0-preview.1',
+        'Content-Length': body.length
+    });
+    var options = {
+        base: 'https://marketplace.visualstudio.com/_apis/public/gallery',
+        requestOptions: {
+            method: 'POST',
+            gzip: true,
+            headers: headers,
+            body: body
+        }
+    };
+    return remote('/extensionquery', options)
+        .pipe(flatmap(function (stream, f) {
+        var rawResult = f.contents.toString('utf8');
+        var result = JSON.parse(rawResult);
+        var extension = result.results[0].extensions[0];
+        if (!extension) {
+            return error("No such extension: " + extension);
+        }
+        var metadata = {
+            id: extension.extensionId,
+            publisherId: extension.publisher,
+            publisherDisplayName: extension.publisher.displayName
+        };
+        var extensionVersion = extension.versions.filter(function (v) { return v.version === version; })[0];
+        if (!extensionVersion) {
+            return error("No such extension version: " + extensionName + " @ " + version);
+        }
+        var asset = extensionVersion.files.filter(function (f) { return f.assetType === 'Microsoft.VisualStudio.Services.VSIXPackage'; })[0];
+        if (!asset) {
+            return error("No VSIX found for extension version: " + extensionName + " @ " + version);
+        }
+        util.log('Downloading extension:', util.colors.yellow(extensionName + "@" + version), '...');
+        var options = {
+            base: asset.source,
+            requestOptions: {
+                gzip: true,
+                headers: baseHeaders
+            }
+        };
+        return remote('', options)
+            .pipe(flatmap(function (stream) {
+            var packageJsonFilter = filter('package.json', { restore: true });
+            return stream
+                .pipe(vzip.src())
+                .pipe(filter('extension/**'))
+                .pipe(rename(function (p) { return p.dirname = p.dirname.replace(/^extension\/?/, ''); }))
+                .pipe(packageJsonFilter)
+                .pipe(buffer())
+                .pipe(json({ __metadata: metadata }))
+                .pipe(packageJsonFilter.restore);
+        }));
+    }));
+}
+exports.src = src;

build/lib/extensions.ts

+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { through } from 'event-stream';
+import { Stream } from 'stream';
+import assign = require('object-assign');
+import remote = require('gulp-remote-src');
+const flatmap = require('gulp-flatmap');
+const vzip = require('gulp-vinyl-zip');
+const filter = require('gulp-filter');
+const rename = require('gulp-rename');
+const util = require('gulp-util');
+const buffer = require('gulp-buffer');
+const json = require('gulp-json-editor');
+
+function error(err: any): Stream {
+	const result = through();
+	setTimeout(() => result.emit('error', err));
+	return result;
+}
+
+const baseHeaders = {
+	'X-Market-Client-Id': 'VSCode Build',
+	'User-Agent': 'VSCode Build',
+};
+
+export function src(extensionName: string, version: string): Stream {
+	const filterType = 7;
+	const value = extensionName;
+	const criterium = { filterType, value };
+	const criteria = [criterium];
+	const pageNumber = 1;
+	const pageSize = 1;
+	const sortBy = 0;
+	const sortOrder = 0;
+	const flags = 0x1 | 0x2 | 0x80;
+	const assetTypes = ['Microsoft.VisualStudio.Services.VSIXPackage'];
+	const filters = [{ criteria, pageNumber, pageSize, sortBy, sortOrder }];
+	const body = JSON.stringify({ filters, assetTypes, flags });
+	const headers: any = assign({}, baseHeaders, {
+		'Content-Type': 'application/json',
+		'Accept': 'application/json;api-version=3.0-preview.1',
+		'Content-Length': body.length
+	});
+
+	const options = {
+		base: 'https://marketplace.visualstudio.com/_apis/public/gallery',
+		requestOptions: {
+			method: 'POST',
+			gzip: true,
+			headers,
+			body: body
+		}
+	};
+
+	return remote('/extensionquery', options)
+		.pipe(flatmap((stream, f) => {
+			const rawResult = f.contents.toString('utf8');
+			const result = JSON.parse(rawResult);
+			const extension = result.results[0].extensions[0];
+			if (!extension) {
+				return error(`No such extension: ${ extension }`);
+			}
+
+			const metadata = {
+				id: extension.extensionId,
+				publisherId: extension.publisher,
+				publisherDisplayName: extension.publisher.displayName
+			};
+
+			const extensionVersion = extension.versions.filter(v => v.version === version)[0];
+			if (!extensionVersion) {
+				return error(`No such extension version: ${ extensionName } @ ${ version }`);
+			}
+
+			const asset = extensionVersion.files.filter(f => f.assetType === 'Microsoft.VisualStudio.Services.VSIXPackage')[0];
+			if (!asset) {
+				return error(`No VSIX found for extension version: ${ extensionName } @ ${ version }`);
+			}
+
+			util.log('Downloading extension:', util.colors.yellow(`${ extensionName }@${ version }`), '...');
+
+			const options = {
+				base: asset.source,
+				requestOptions: {
+					gzip: true,
+					headers: baseHeaders
+				}
+			};
+
+			return remote('', options)
+				.pipe(flatmap(stream => {
+					const packageJsonFilter = filter('package.json', { restore: true });
+
+					return stream
+						.pipe(vzip.src())
+						.pipe(filter('extension/**'))
+						.pipe(rename(p => p.dirname = p.dirname.replace(/^extension\/?/, '')))
+						.pipe(packageJsonFilter)
+						.pipe(buffer())
+						.pipe(json({ __metadata: metadata }))
+						.pipe(packageJsonFilter.restore);
+				}));
+		}));
+}

build/lib/typings/gulp-remote-src.d.ts

+declare module 'gulp-remote-src' {
+
+	import stream = require("stream");
+
+	function remote(url: string, options: remote.IOptions): stream.Stream;
+
+	module remote {
+		export interface IRequestOptions {
+			body?: any;
+			json?: boolean;
+			method?: string;
+			headers?: any;
+		}
+
+		export interface IOptions {
+			base?: string;
+			buffer?: boolean;
+			requestOptions?: IRequestOptions;
+		}
+	}
+
+	export = remote;
+}
\ No newline at end of file