Merge branch 'joao/build-debug'

build/azure-pipelines/product-compile.yml

@@ -82,11 +82,18 @@ steps:
 
 - script: |
     set -e
-    node build/azure-pipelines/common/installDistroDependencies.js
-    node build/azure-pipelines/common/installDistroDependencies.js remote
+    CHILD_CONCURRENCY=1 node build/azure-pipelines/common/installDistroDependencies.js
+    CHILD_CONCURRENCY=1 node build/azure-pipelines/common/installDistroDependencies.js remote
   displayName: Install distro dependencies
   condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true'))
 
+# Mixin must run before optimize, because the CSS loader will
+# inline small SVGs
+- script: |
+    set -e
+    yarn gulp mixin
+  displayName: Mix in quality
+
 - script: |
     set -e
     yarn gulp hygiene

build/gulpfile.extensions.js

@@ -153,10 +153,11 @@ gulp.task(compileExtensionsBuildLegacyTask);
 // Azure Pipelines
 
 const cleanExtensionsBuildTask = task.define('clean-extensions-build', util.rimraf('.build/extensions'));
-const compileExtensionsBuildTask = task.define('compile-extensions-build', task.series(cleanExtensionsBuildTask, () => {
-	return ext.packageExtensionsStream()
-		.pipe(gulp.dest('.build'));
-}));
+const compileExtensionsBuildTask = task.define('compile-extensions-build', task.series(
+	cleanExtensionsBuildTask,
+	task.define('bundle-extensions-build', () => ext.packageLocalExtensionsStream().pipe(gulp.dest('.build'))),
+	task.define('bundle-marketplace-extensions-build', () => ext.packageMarketplaceExtensionsStream().pipe(gulp.dest('.build'))),
+));
 
 gulp.task(compileExtensionsBuildTask);
 exports.compileExtensionsBuildTask = compileExtensionsBuildTask;
\ No newline at end of file

build/lib/extensions.js

@@ -81,7 +81,7 @@ function fromLocalWebpack(extensionPath) {
             return data;
         }))
             .pipe(packageJsonFilter.restore);
-        const webpackStreams = webpackConfigLocations.map(webpackConfigPath => () => {
+        const webpackStreams = webpackConfigLocations.map(webpackConfigPath => {
             const webpackDone = (err, stats) => {
                 fancyLog(`Bundled extension: ${ansiColors.yellow(path.join(path.basename(extensionPath), path.relative(extensionPath, webpackConfigPath)))}...`);
                 if (err) {
@@ -120,7 +120,7 @@ function fromLocalWebpack(extensionPath) {
                 this.emit('data', data);
             }));
         });
-        es.merge(sequence(webpackStreams), patchFilesStream)
+        es.merge(...webpackStreams, patchFilesStream)
             // .pipe(es.through(function (data) {
             // 	// debug
             // 	console.log('out', data.path, data.contents.length);
@@ -186,29 +186,7 @@ const excludedExtensions = [
     'ms-vscode.node-debug2',
 ];
 const builtInExtensions = require('../builtInExtensions.json');
-/**
- * We're doing way too much stuff at once, with webpack et al. So much stuff
- * that while downloading extensions from the marketplace, node js doesn't get enough
- * stack frames to complete the download in under 2 minutes, at which point the
- * marketplace server cuts off the http request. So, we sequentialize the extensino tasks.
- */
-function sequence(streamProviders) {
-    const result = es.through();
-    function pop() {
-        if (streamProviders.length === 0) {
-            result.emit('end');
-        }
-        else {
-            const fn = streamProviders.shift();
-            fn()
-                .on('end', function () { setTimeout(pop, 0); })
-                .pipe(result, { end: false });
-        }
-    }
-    pop();
-    return result;
-}
-function packageExtensionsStream() {
+function packageLocalExtensionsStream() {
     const localExtensionDescriptions = glob.sync('extensions/*/package.json')
         .map(manifestPath => {
         const extensionPath = path.dirname(path.join(root, manifestPath));
@@ -217,18 +195,20 @@ function packageExtensionsStream() {
     })
         .filter(({ name }) => excludedExtensions.indexOf(name) === -1)
         .filter(({ name }) => builtInExtensions.every(b => b.name !== name));
-    const localExtensions = () => sequence([...localExtensionDescriptions.map(extension => () => {
-            return fromLocal(extension.path)
-                .pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`));
-        })]);
-    const localExtensionDependencies = () => gulp.src('extensions/node_modules/**', { base: '.' });
-    const marketplaceExtensions = () => es.merge(...builtInExtensions
-        .map(extension => {
+    return es.merge(gulp.src('extensions/node_modules/**', { base: '.' }), ...localExtensionDescriptions.map(extension => {
+        return fromLocal(extension.path)
+            .pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`));
+    }))
+        .pipe(util2.setExecutableBit(['**/*.sh']))
+        .pipe(filter(['**', '!**/*.js.map']));
+}
+exports.packageLocalExtensionsStream = packageLocalExtensionsStream;
+function packageMarketplaceExtensionsStream() {
+    return es.merge(builtInExtensions.map(extension => {
         return fromMarketplace(extension.name, extension.version, extension.metadata)
             .pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`));
-    }));
-    return sequence([localExtensions, localExtensionDependencies, marketplaceExtensions])
+    }))
         .pipe(util2.setExecutableBit(['**/*.sh']))
         .pipe(filter(['**', '!**/*.js.map']));
 }
-exports.packageExtensionsStream = packageExtensionsStream;
+exports.packageMarketplaceExtensionsStream = packageMarketplaceExtensionsStream;

build/lib/extensions.ts

@@ -93,7 +93,7 @@ function fromLocalWebpack(extensionPath: string): Stream {
 			.pipe(packageJsonFilter.restore);
 
 
-		const webpackStreams = webpackConfigLocations.map(webpackConfigPath => () => {
+		const webpackStreams = webpackConfigLocations.map(webpackConfigPath => {
 
 			const webpackDone = (err: any, stats: any) => {
 				fancyLog(`Bundled extension: ${ansiColors.yellow(path.join(path.basename(extensionPath), path.relative(extensionPath, webpackConfigPath)))}...`);
@@ -140,7 +140,7 @@ function fromLocalWebpack(extensionPath: string): Stream {
 				}));
 		});
 
-		es.merge(sequence(webpackStreams), patchFilesStream)
+		es.merge(...webpackStreams, patchFilesStream)
 			// .pipe(es.through(function (data) {
 			// 	// debug
 			// 	console.log('out', data.path, data.contents.length);
@@ -227,31 +227,7 @@ interface IBuiltInExtension {
 
 const builtInExtensions: IBuiltInExtension[] = require('../builtInExtensions.json');
 
-/**
- * We're doing way too much stuff at once, with webpack et al. So much stuff
- * that while downloading extensions from the marketplace, node js doesn't get enough
- * stack frames to complete the download in under 2 minutes, at which point the
- * marketplace server cuts off the http request. So, we sequentialize the extensino tasks.
- */
-function sequence(streamProviders: { (): Stream }[]): Stream {
-	const result = es.through();
-
-	function pop() {
-		if (streamProviders.length === 0) {
-			result.emit('end');
-		} else {
-			const fn = streamProviders.shift()!;
-			fn()
-				.on('end', function () { setTimeout(pop, 0); })
-				.pipe(result, { end: false });
-		}
-	}
-
-	pop();
-	return result;
-}
-
-export function packageExtensionsStream(): NodeJS.ReadWriteStream {
+export function packageLocalExtensionsStream(): NodeJS.ReadWriteStream {
 	const localExtensionDescriptions = (<string[]>glob.sync('extensions/*/package.json'))
 		.map(manifestPath => {
 			const extensionPath = path.dirname(path.join(root, manifestPath));
@@ -261,22 +237,22 @@ export function packageExtensionsStream(): NodeJS.ReadWriteStream {
 		.filter(({ name }) => excludedExtensions.indexOf(name) === -1)
 		.filter(({ name }) => builtInExtensions.every(b => b.name !== name));
 
-	const localExtensions = () => sequence([...localExtensionDescriptions.map(extension => () => {
-		return fromLocal(extension.path)
-			.pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`));
-	})]);
-
-	const localExtensionDependencies = () => gulp.src('extensions/node_modules/**', { base: '.' });
-
-	const marketplaceExtensions = () => es.merge(
-		...builtInExtensions
-			.map(extension => {
-				return fromMarketplace(extension.name, extension.version, extension.metadata)
-					.pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`));
-			})
-	);
+	return es.merge(
+		gulp.src('extensions/node_modules/**', { base: '.' }),
+		...localExtensionDescriptions.map(extension => {
+			return fromLocal(extension.path)
+				.pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`));
+		})
+	)
+		.pipe(util2.setExecutableBit(['**/*.sh']))
+		.pipe(filter(['**', '!**/*.js.map']));
+}
 
-	return sequence([localExtensions, localExtensionDependencies, marketplaceExtensions])
+export function packageMarketplaceExtensionsStream(): NodeJS.ReadWriteStream {
+	return es.merge(builtInExtensions.map(extension => {
+		return fromMarketplace(extension.name, extension.version, extension.metadata)
+			.pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`));
+	}))
 		.pipe(util2.setExecutableBit(['**/*.sh']))
 		.pipe(filter(['**', '!**/*.js.map']));
 }