Merge pull request #57956 from Microsoft/joh/vsce-next

use packagedDependencies from running vsce#ls

Merge pull request #57956 from Microsoft/joh/vsce-next
use packagedDependencies from running vsce#ls
74fa1d89 · Johannes Rieken · GitHub · 126082fd · 9536dde3 · 74fa1d89
10 changed file
--- a/build/gulpfile.vscode.js
+++ b/build/gulpfile.vscode.js
@@ -327,15 +327,16 @@ function packageTask(platform, arch, opts) {
 				.pipe(rename('bin/' + product.applicationName)));
 		}

-		return result.pipe(es.through(undefined, function () {
-			// submit all stats that have been collected during the build phase
-			if (!opts.stats) {
-				return this.emit('end');
-			}
-			const { submitAllStats } = require('./lib/stats');
-			submitAllStats(product).then(() => this.emit('end')).catch(() => this.emit('end'));
-
-		})).pipe(vfs.dest(destination));
+		// submit all stats that have been collected
+		// during the build phase
+		if (opts.stats) {
+			result.on('end', () => {
+				const { submitAllStats } = require('./lib/stats');
+				submitAllStats(product).then(() => console.log('Submitted bundle stats!'));
+			});
+		}
+
+		return result.pipe(vfs.dest(destination));
 	};
 }


--- a/build/lib/extensions.js
+++ b/build/lib/extensions.js
@@ -37,8 +37,26 @@ var webpack = require('webpack');
 var webpackGulp = require('webpack-stream');
 var root = path.resolve(path.join(__dirname, '..', '..'));
 function fromLocal(extensionPath, sourceMappingURLBase) {
+    var webpackFilename = path.join(extensionPath, 'extension.webpack.config.js');
+    if (fs.existsSync(webpackFilename)) {
+        return fromLocalWebpack(extensionPath, sourceMappingURLBase);
+    }
+    else {
+        return fromLocalNormal(extensionPath);
+    }
+}
+exports.fromLocal = fromLocal;
+function fromLocalWebpack(extensionPath, sourceMappingURLBase) {
    var result = es.through();
-    vsce.listFiles({ cwd: extensionPath, packageManager: vsce.PackageManager.Yarn }).then(function (fileNames) {
+    var packagedDependencies = [];
+    var packageJsonConfig = require(path.join(extensionPath, 'package.json'));
+    var webpackRootConfig = require(path.join(extensionPath, 'extension.webpack.config.js'));
+    for (var key in webpackRootConfig.externals) {
+        if (key in packageJsonConfig.dependencies) {
+            packagedDependencies.push(key);
+        }
+    }
+    vsce.listFiles({ cwd: extensionPath, packageManager: vsce.PackageManager.Yarn, packagedDependencies: packagedDependencies }).then(function (fileNames) {
        var files = fileNames
            .map(function (fileName) { return path.join(extensionPath, fileName); })
            .map(function (filePath) { return new File({
@@ -51,76 +69,86 @@ function fromLocal(extensionPath, sourceMappingURLBase) {
        // check for a webpack configuration files, then invoke webpack
        // and merge its output with the files stream. also rewrite the package.json
        // file to a new entry point
-        var pattern = path.join(extensionPath, '/**/extension.webpack.config.js');
-        var webpackConfigLocations = glob.sync(pattern, { ignore: ['**/node_modules'] });
-        if (webpackConfigLocations.length) {
-            var packageJsonFilter = filter(function (f) {
-                if (path.basename(f.path) === 'package.json') {
-                    // only modify package.json's next to the webpack file.
-                    // to be safe, use existsSync instead of path comparison.
-                    return fs.existsSync(path.join(path.dirname(f.path), 'extension.webpack.config.js'));
+        var webpackConfigLocations = glob.sync(path.join(extensionPath, '/**/extension.webpack.config.js'), { ignore: ['**/node_modules'] });
+        var packageJsonFilter = filter(function (f) {
+            if (path.basename(f.path) === 'package.json') {
+                // only modify package.json's next to the webpack file.
+                // to be safe, use existsSync instead of path comparison.
+                return fs.existsSync(path.join(path.dirname(f.path), 'extension.webpack.config.js'));
+            }
+            return false;
+        }, { restore: true });
+        var patchFilesStream = filesStream
+            .pipe(packageJsonFilter)
+            .pipe(buffer())
+            .pipe(json(function (data) {
+            // hardcoded entry point directory!
+            data.main = data.main.replace('/out/', /dist/);
+            return data;
+        }))
+            .pipe(packageJsonFilter.restore);
+        var webpackStreams = webpackConfigLocations.map(function (webpackConfigPath) {
+            var webpackDone = function (err, stats) {
+                util.log("Bundled extension: " + util.colors.yellow(path.join(path.basename(extensionPath), path.relative(extensionPath, webpackConfigPath))) + "...");
+                if (err) {
+                    result.emit('error', err);
                }
-                return false;
-            }, { restore: true });
-            var patchFilesStream = filesStream
-                .pipe(packageJsonFilter)
-                .pipe(buffer())
-                .pipe(json(function (data) {
-                // hardcoded entry point directory!
-                data.main = data.main.replace('/out/', /dist/);
-                return data;
+                var compilation = stats.compilation;
+                if (compilation.errors.length > 0) {
+                    result.emit('error', compilation.errors.join('\n'));
+                }
+                if (compilation.warnings.length > 0) {
+                    result.emit('error', compilation.warnings.join('\n'));
+                }
+            };
+            var webpackConfig = __assign({}, require(webpackConfigPath), { mode: 'production' });
+            var relativeOutputPath = path.relative(extensionPath, webpackConfig.output.path);
+            return webpackGulp(webpackConfig, webpack, webpackDone)
+                .pipe(es.through(function (data) {
+                data.stat = data.stat || {};
+                data.base = extensionPath;
+                this.emit('data', data);
            }))
-                .pipe(packageJsonFilter.restore);
-            var webpackStreams = webpackConfigLocations.map(function (webpackConfigPath) {
-                var webpackDone = function (err, stats) {
-                    util.log("Bundled extension: " + util.colors.yellow(path.join(path.basename(extensionPath), path.relative(extensionPath, webpackConfigPath))) + "...");
-                    if (err) {
-                        result.emit('error', err);
-                    }
-                    var compilation = stats.compilation;
-                    if (compilation.errors.length > 0) {
-                        result.emit('error', compilation.errors.join('\n'));
-                    }
-                    if (compilation.warnings.length > 0) {
-                        result.emit('error', compilation.warnings.join('\n'));
-                    }
-                };
-                var webpackConfig = __assign({}, require(webpackConfigPath), { mode: 'production' });
-                var relativeOutputPath = path.relative(extensionPath, webpackConfig.output.path);
-                return webpackGulp(webpackConfig, webpack, webpackDone)
-                    .pipe(es.through(function (data) {
-                    data.stat = data.stat || {};
-                    data.base = extensionPath;
-                    this.emit('data', data);
-                }))
-                    .pipe(es.through(function (data) {
-                    // source map handling:
-                    // * rewrite sourceMappingURL
-                    // * save to disk so that upload-task picks this up
-                    if (sourceMappingURLBase) {
-                        var contents = data.contents.toString('utf8');
-                        data.contents = Buffer.from(contents.replace(/\n\/\/# sourceMappingURL=(.*)$/gm, function (_m, g1) {
-                            return "\n//# sourceMappingURL=" + sourceMappingURLBase + "/extensions/" + path.basename(extensionPath) + "/" + relativeOutputPath + "/" + g1;
-                        }), 'utf8');
-                        if (/\.js\.map$/.test(data.path)) {
-                            if (!fs.existsSync(path.dirname(data.path))) {
-                                fs.mkdirSync(path.dirname(data.path));
-                            }
-                            fs.writeFileSync(data.path, data.contents);
+                .pipe(es.through(function (data) {
+                // source map handling:
+                // * rewrite sourceMappingURL
+                // * save to disk so that upload-task picks this up
+                if (sourceMappingURLBase) {
+                    var contents = data.contents.toString('utf8');
+                    data.contents = Buffer.from(contents.replace(/\n\/\/# sourceMappingURL=(.*)$/gm, function (_m, g1) {
+                        return "\n//# sourceMappingURL=" + sourceMappingURLBase + "/extensions/" + path.basename(extensionPath) + "/" + relativeOutputPath + "/" + g1;
+                    }), 'utf8');
+                    if (/\.js\.map$/.test(data.path)) {
+                        if (!fs.existsSync(path.dirname(data.path))) {
+                            fs.mkdirSync(path.dirname(data.path));
                        }
+                        fs.writeFileSync(data.path, data.contents);
                    }
-                    this.emit('data', data);
-                }));
-            });
-            es.merge.apply(es, webpackStreams.concat([patchFilesStream])).pipe(result);
-        }
-        else {
-            filesStream.pipe(result);
-        }
+                }
+                this.emit('data', data);
+            }));
+        });
+        es.merge.apply(es, webpackStreams.concat([patchFilesStream])).pipe(result);
    }).catch(function (err) { return result.emit('error', err); });
    return result.pipe(stats_1.createStatsStream(path.basename(extensionPath)));
 }
-exports.fromLocal = fromLocal;
+function fromLocalNormal(extensionPath) {
+    var result = es.through();
+    vsce.listFiles({ cwd: extensionPath, packageManager: vsce.PackageManager.Yarn })
+        .then(function (fileNames) {
+        var files = fileNames
+            .map(function (fileName) { return path.join(extensionPath, fileName); })
+            .map(function (filePath) { return new File({
+            path: filePath,
+            stat: fs.statSync(filePath),
+            base: extensionPath,
+            contents: fs.createReadStream(filePath)
+        }); });
+        es.readArray(files).pipe(result);
+    })
+        .catch(function (err) { return result.emit('error', err); });
+    return result;
+}
 function error(err) {
    var result = es.through();
    setTimeout(function () { return result.emit('error', err); });

--- a/build/lib/extensions.ts
+++ b/build/lib/extensions.ts
@@ -28,9 +28,27 @@ const webpackGulp = require('webpack-stream');
 const root = path.resolve(path.join(__dirname, '..', '..'));

 export function fromLocal(extensionPath: string, sourceMappingURLBase?: string): Stream {
+	const webpackFilename = path.join(extensionPath, 'extension.webpack.config.js');
+	if (fs.existsSync(webpackFilename)) {
+		return fromLocalWebpack(extensionPath, sourceMappingURLBase);
+	} else {
+		return fromLocalNormal(extensionPath);
+	}
+}
+
+function fromLocalWebpack(extensionPath: string, sourceMappingURLBase: string): Stream {
 	let result = es.through();

-	vsce.listFiles({ cwd: extensionPath, packageManager: vsce.PackageManager.Yarn }).then(fileNames => {
+	let packagedDependencies: string[] = [];
+	let packageJsonConfig = require(path.join(extensionPath, 'package.json'));
+	let webpackRootConfig = require(path.join(extensionPath, 'extension.webpack.config.js'));
+	for (const key in webpackRootConfig.externals) {
+		if (key in packageJsonConfig.dependencies) {
+			packagedDependencies.push(key);
+		}
+	}
+
+	vsce.listFiles({ cwd: extensionPath, packageManager: vsce.PackageManager.Yarn, packagedDependencies }).then(fileNames => {
 		const files = fileNames
 			.map(fileName => path.join(extensionPath, fileName))
 			.map(filePath => new File({
@@ -45,96 +63,114 @@ export function fromLocal(extensionPath: string, sourceMappingURLBase?: string):
 		// check for a webpack configuration files, then invoke webpack
 		// and merge its output with the files stream. also rewrite the package.json
 		// file to a new entry point
-		const pattern = path.join(extensionPath, '/**/extension.webpack.config.js');
-		const webpackConfigLocations = (<string[]>glob.sync(pattern, { ignore: ['**/node_modules'] }));
-		if (webpackConfigLocations.length) {
-
-			const packageJsonFilter = filter(f => {
-				if (path.basename(f.path) === 'package.json') {
-					// only modify package.json's next to the webpack file.
-					// to be safe, use existsSync instead of path comparison.
-					return fs.existsSync(path.join(path.dirname(f.path), 'extension.webpack.config.js'));
+		const webpackConfigLocations = (<string[]>glob.sync(
+			path.join(extensionPath, '/**/extension.webpack.config.js'),
+			{ ignore: ['**/node_modules'] }
+		));
+
+		const packageJsonFilter = filter(f => {
+			if (path.basename(f.path) === 'package.json') {
+				// only modify package.json's next to the webpack file.
+				// to be safe, use existsSync instead of path comparison.
+				return fs.existsSync(path.join(path.dirname(f.path), 'extension.webpack.config.js'));
+			}
+			return false;
+		}, { restore: true });
+
+		const patchFilesStream = filesStream
+			.pipe(packageJsonFilter)
+			.pipe(buffer())
+			.pipe(json(data => {
+				// hardcoded entry point directory!
+				data.main = data.main.replace('/out/', /dist/);
+				return data;
+			}))
+			.pipe(packageJsonFilter.restore);
+
+
+		const webpackStreams = webpackConfigLocations.map(webpackConfigPath => {
+
+			const webpackDone = (err, stats) => {
+				util.log(`Bundled extension: ${util.colors.yellow(path.join(path.basename(extensionPath), path.relative(extensionPath, webpackConfigPath)))}...`);
+				if (err) {
+					result.emit('error', err);
 				}
-				return false;
-			}, { restore: true });
-
-			const patchFilesStream = filesStream
-				.pipe(packageJsonFilter)
-				.pipe(buffer())
-				.pipe(json(data => {
-					// hardcoded entry point directory!
-					data.main = data.main.replace('/out/', /dist/);
-					return data;
-				}))
-				.pipe(packageJsonFilter.restore);
-
+				const { compilation } = stats;
+				if (compilation.errors.length > 0) {
+					result.emit('error', compilation.errors.join('\n'));
+				}
+				if (compilation.warnings.length > 0) {
+					result.emit('error', compilation.warnings.join('\n'));
+				}
+			};

-			const webpackStreams = webpackConfigLocations.map(webpackConfigPath => {
+			const webpackConfig = {
+				...require(webpackConfigPath),
+				...{ mode: 'production' }
+			};
+			let relativeOutputPath = path.relative(extensionPath, webpackConfig.output.path);

-				const webpackDone = (err, stats) => {
-					util.log(`Bundled extension: ${util.colors.yellow(path.join(path.basename(extensionPath), path.relative(extensionPath, webpackConfigPath)))}...`);
-					if (err) {
-						result.emit('error', err);
-					}
-					const { compilation } = stats;
-					if (compilation.errors.length > 0) {
-						result.emit('error', compilation.errors.join('\n'));
-					}
-					if (compilation.warnings.length > 0) {
-						result.emit('error', compilation.warnings.join('\n'));
-					}
-				};
-
-				const webpackConfig = {
-					...require(webpackConfigPath),
-					...{ mode: 'production' }
-				};
-				let relativeOutputPath = path.relative(extensionPath, webpackConfig.output.path);
-
-				return webpackGulp(webpackConfig, webpack, webpackDone)
-					.pipe(es.through(function (data) {
-						data.stat = data.stat || {};
-						data.base = extensionPath;
-						this.emit('data', data);
-					}))
-					.pipe(es.through(function (data: File) {
-						// source map handling:
-						// * rewrite sourceMappingURL
-						// * save to disk so that upload-task picks this up
-						if (sourceMappingURLBase) {
-							const contents = (<Buffer>data.contents).toString('utf8');
-							data.contents = Buffer.from(contents.replace(/\n\/\/# sourceMappingURL=(.*)$/gm, function (_m, g1) {
-								return `\n//# sourceMappingURL=${sourceMappingURLBase}/extensions/${path.basename(extensionPath)}/${relativeOutputPath}/${g1}`;
-							}), 'utf8');
-
-							if (/\.js\.map$/.test(data.path)) {
-								if (!fs.existsSync(path.dirname(data.path))) {
-									fs.mkdirSync(path.dirname(data.path));
-								}
-								fs.writeFileSync(data.path, data.contents);
+			return webpackGulp(webpackConfig, webpack, webpackDone)
+				.pipe(es.through(function (data) {
+					data.stat = data.stat || {};
+					data.base = extensionPath;
+					this.emit('data', data);
+				}))
+				.pipe(es.through(function (data: File) {
+					// source map handling:
+					// * rewrite sourceMappingURL
+					// * save to disk so that upload-task picks this up
+					if (sourceMappingURLBase) {
+						const contents = (<Buffer>data.contents).toString('utf8');
+						data.contents = Buffer.from(contents.replace(/\n\/\/# sourceMappingURL=(.*)$/gm, function (_m, g1) {
+							return `\n//# sourceMappingURL=${sourceMappingURLBase}/extensions/${path.basename(extensionPath)}/${relativeOutputPath}/${g1}`;
+						}), 'utf8');
+
+						if (/\.js\.map$/.test(data.path)) {
+							if (!fs.existsSync(path.dirname(data.path))) {
+								fs.mkdirSync(path.dirname(data.path));
 							}
+							fs.writeFileSync(data.path, data.contents);
 						}
-						this.emit('data', data);
-					}));
-			});
-
-			es.merge(...webpackStreams, patchFilesStream)
-				// .pipe(es.through(function (data) {
-				// 	// debug
-				// 	console.log('out', data.path, data.contents.length);
-				// 	this.emit('data', data);
-				// }))
-				.pipe(result);
-
-		} else {
-			filesStream.pipe(result);
-		}
+					}
+					this.emit('data', data);
+				}));
+		});
+
+		es.merge(...webpackStreams, patchFilesStream)
+			// .pipe(es.through(function (data) {
+			// 	// debug
+			// 	console.log('out', data.path, data.contents.length);
+			// 	this.emit('data', data);
+			// }))
+			.pipe(result);

 	}).catch(err => result.emit('error', err));

 	return result.pipe(createStatsStream(path.basename(extensionPath)));
 }

+function fromLocalNormal(extensionPath: string): Stream {
+	const result = es.through();
+
+	vsce.listFiles({ cwd: extensionPath, packageManager: vsce.PackageManager.Yarn })
+		.then(fileNames => {
+			const files = fileNames
+				.map(fileName => path.join(extensionPath, fileName))
+				.map(filePath => new File({
+					path: filePath,
+					stat: fs.statSync(filePath),
+					base: extensionPath,
+					contents: fs.createReadStream(filePath) as any
+				}));
+
+			es.readArray(files).pipe(result);
+		})
+		.catch(err => result.emit('error', err));
+
+	return result;
+}
+
 function error(err: any): Stream {
 	const result = es.through();
 	setTimeout(() => result.emit('error', err));

--- a/build/lib/stats.js
+++ b/build/lib/stats.js
@@ -105,14 +105,15 @@ function submitAllStats(productJson) {
            .setAutoCollectPerformance(false)
            .setAutoCollectRequests(false)
            .start();
-        appInsights.defaultClient.config.endpointUrl = 'https://vortex.data.microsoft.com/collect/v1';
+        var client = appInsights.getClient(productJson.aiConfig.asimovKey);
+        client.config.endpointUrl = 'https://vortex.data.microsoft.com/collect/v1';
        /* __GDPR__
            "monacoworkbench/bundleStats" : {
                "outcome" : {"classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "isMeasurement": true }
            }
        */
-        appInsights.defaultClient.trackEvent("monacoworkbench/bundleStats", undefined, measurements);
-        appInsights.defaultClient.sendPendingData(function () { return resolve(); });
+        client.trackEvent("monacoworkbench/bundleStats", undefined, measurements);
+        client.sendPendingData(function () { return resolve(); });
    });
 }
 exports.submitAllStats = submitAllStats;
--- a/build/lib/stats.ts
+++ b/build/lib/stats.ts
@@ -111,14 +111,16 @@ export function submitAllStats(productJson: any): Promise<void> {
 			.setAutoCollectRequests(false)
 			.start();

-		appInsights.defaultClient.config.endpointUrl = 'https://vortex.data.microsoft.com/collect/v1';
+		const client = appInsights.getClient(productJson.aiConfig.asimovKey);
+		client.config.endpointUrl = 'https://vortex.data.microsoft.com/collect/v1';
+
 		/* __GDPR__
 			"monacoworkbench/bundleStats" : {
 				"outcome" : {"classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "isMeasurement": true }
 			}
 		*/
-		appInsights.defaultClient.trackEvent(`monacoworkbench/bundleStats`, undefined, measurements);
-		appInsights.defaultClient.sendPendingData(() => resolve());
+		client.trackEvent(`monacoworkbench/bundleStats`, undefined, measurements);
+		client.sendPendingData(() => resolve());
 	});

 }
--- a/extensions/emmet/.vscodeignore
+++ b/extensions/emmet/.vscodeignore
@@ -2,7 +2,5 @@ test/**
 src/**
 out/**
 tsconfig.json
-node_modules/@emmetio/css-parser/**
-node_modules/@emmetio/html-matcher/**
-node_modules/@emmetio/math-expression/**
-node_modules/image-size/**
+extension.webpack.config.js
+
--- a/extensions/git/.vscodeignore
+++ b/extensions/git/.vscodeignore
@@ -3,8 +3,4 @@ test/**
 out/**
 tsconfig.json
 build/**
-node_modules/byline/**
-node_modules/file-type/**
-node_modules/iconv-lite/**
-node_modules/jschardet/**
-node_modules/which/**
+extension.webpack.config.js
--- a/extensions/typescript-language-features/.vscodeignore
+++ b/extensions/typescript-language-features/.vscodeignore
@@ -2,7 +2,5 @@ build/**
 src/**
 test/**
 out/**
-node_modules/jsonc-parser/**
-node_modules/semver/**
 tsconfig.json
 extension.webpack.config.js
--- a/package.json
+++ b/package.json
@@ -127,7 +127,7 @@
    "underscore": "^1.8.2",
    "vinyl": "^0.4.5",
    "vinyl-fs": "^2.4.3",
-    "vsce": "1.33.2",
+    "vsce": "1.48.0",
    "vscode-nls-dev": "3.2.2",
    "webpack": "^4.16.5",
    "webpack-cli": "^3.1.0",

--- a/yarn.lock
+++ b/yarn.lock
@@ -4568,6 +4568,10 @@ lodash@^4.0.0, lodash@^4.13.1, lodash@^4.14.0, lodash@^4.15.0, lodash@^4.3.0:
  version "4.17.4"
  resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae"

+lodash@^4.17.10:
+  version "4.17.10"
+  resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.10.tgz#1b7793cf7259ea38fb3661d4d38b3260af8ae4e7"
+
 lodash@~1.0.1:
  version "1.0.2"
  resolved "https://registry.yarnpkg.com/lodash/-/lodash-1.0.2.tgz#8f57560c83b59fc270bd3d561b690043430e2551"
@@ -7855,15 +7859,15 @@ vm-browserify@0.0.4:
  dependencies:
    indexof "0.0.1"

-vsce@1.33.2:
-  version "1.33.2"
-  resolved "https://registry.yarnpkg.com/vsce/-/vsce-1.33.2.tgz#3645f69aaf984e22f74ea49d35f38dd18d66ff5f"
+vsce@1.48.0:
+  version "1.48.0"
+  resolved "https://registry.yarnpkg.com/vsce/-/vsce-1.48.0.tgz#31c1a4c6909c3b8bdc48b3d32cc8c8e94c7113a2"
  dependencies:
    cheerio "^1.0.0-rc.1"
    commander "^2.8.1"
    denodeify "^1.2.1"
    glob "^7.0.6"
-    lodash "^4.15.0"
+    lodash "^4.17.10"
    markdown-it "^8.3.1"
    mime "^1.3.4"
    minimatch "^3.0.3"
@@ -7873,7 +7877,7 @@ vsce@1.33.2:
    semver "^5.1.0"
    tmp "0.0.29"
    url-join "^1.1.0"
-    vso-node-api "^6.1.2-preview"
+    vso-node-api "6.1.2-preview"
    yauzl "^2.3.1"
    yazl "^2.2.2"

@@ -7949,7 +7953,7 @@ vscode-xterm@3.8.0-beta1:
  version "3.8.0-beta1"
  resolved "https://registry.yarnpkg.com/vscode-xterm/-/vscode-xterm-3.8.0-beta1.tgz#d4e7810bc61c9798f353832844974c18f213c053"

-vso-node-api@^6.1.2-preview:
+vso-node-api@6.1.2-preview:
  version "6.1.2-preview"
  resolved "https://registry.yarnpkg.com/vso-node-api/-/vso-node-api-6.1.2-preview.tgz#aab3546df2451ecd894e071bb99b5df19c5fa78f"
  dependencies: