Merge pull request #56261 from Microsoft/joh/extpack

Use webpack for emmet and git extension

.gitignore

@@ -3,6 +3,7 @@ npm-debug.log
 Thumbs.db
 node_modules/
 .build/
+extensions/**/dist/
 out/
 out-build/
 out-editor/
@@ -17,4 +18,4 @@ build/node_modules
 coverage/
 test_data/
 test-results/
-yarn-error.log
\ No newline at end of file
+yarn-error.log

build/gulpfile.extensions.js

@@ -89,7 +89,7 @@ const tasks = compilations.map(function (tsconfigFile) {
 				.pipe(tsFilter)
 				.pipe(util.loadSourcemaps())
 				.pipe(compilation())
-				.pipe(build ? nlsDev.rewriteLocalizeCalls() : es.through())
+				.pipe(build ? nlsDev.rewriteLocalizeCalls({ keepFilenames: true }) : es.through())
 				.pipe(build ? util.stripSourceMappingURL() : es.through())
 				.pipe(sourcemaps.write('.', {
 					sourceMappingURL: !build ? null : f => `${baseUrl}/${f.relative}.map`,
@@ -167,4 +167,4 @@ gulp.task('watch-extensions', tasks.map(t => t.watch));
 
 gulp.task('clean-extensions-build', tasks.map(t => t.cleanBuild));
 gulp.task('compile-extensions-build', tasks.map(t => t.compileBuild));
-gulp.task('watch-extensions-build', tasks.map(t => t.watchBuild));
\ No newline at end of file
+gulp.task('watch-extensions-build', tasks.map(t => t.watchBuild));

build/gulpfile.vscode.js

@@ -113,9 +113,9 @@ gulp.task('optimize-index-js', ['optimize-vscode'], () => {
 	fs.writeFileSync(fullpath, newContents);
 });
 
-const baseUrl = `https://ticino.blob.core.windows.net/sourcemaps/${commit}/core`;
+const sourceMappingURLBase = `https://ticino.blob.core.windows.net/sourcemaps/${commit}`;
 gulp.task('clean-minified-vscode', util.rimraf('out-vscode-min'));
-gulp.task('minify-vscode', ['clean-minified-vscode', 'optimize-index-js'], common.minifyTask('out-vscode', baseUrl));
+gulp.task('minify-vscode', ['clean-minified-vscode', 'optimize-index-js'], common.minifyTask('out-vscode', `${sourceMappingURLBase}/core`));
 
 // Package
 
@@ -240,7 +240,7 @@ function packageTask(platform, arch, opts) {
 			.filter(({ name }) => builtInExtensions.every(b => b.name !== name));
 
 		const localExtensions = es.merge(...localExtensionDescriptions.map(extension => {
-			return ext.fromLocal(extension.path)
+			return ext.fromLocal(extension.path, sourceMappingURLBase)
 				.pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`));
 		}));
 
@@ -451,16 +451,22 @@ gulp.task('vscode-translations-import', function () {
 
 // Sourcemaps
 
-gulp.task('upload-vscode-sourcemaps', ['minify-vscode'], () => {
+gulp.task('upload-vscode-sourcemaps', ['vscode-darwin-min', 'minify-vscode'], () => {
 	const vs = gulp.src('out-vscode-min/**/*.map', { base: 'out-vscode-min' })
 		.pipe(es.mapSync(f => {
 			f.path = `${f.base}/core/${f.relative}`;
 			return f;
 		}));
 
-	const extensions = gulp.src('extensions/**/out/**/*.map', { base: '.' });
+	const extensionsOut = gulp.src('extensions/**/out/**/*.map', { base: '.' });
+	const extensionsDist = gulp.src('extensions/**/dist/**/*.map', { base: '.' });
 
-	return es.merge(vs, extensions)
+	return es.merge(vs, extensionsOut, extensionsDist)
+		.pipe(es.through(function (data) {
+			// debug
+			console.log('Uploading Sourcemap', data.relative);
+			this.emit('data', data);
+		}))
 		.pipe(azure.upload({
 			account: process.env.AZURE_STORAGE_ACCOUNT,
 			key: process.env.AZURE_STORAGE_ACCESS_KEY,
@@ -505,7 +511,7 @@ function getSettingsSearchBuildId(packageJson) {
 		const branch = process.env.BUILD_SOURCEBRANCH;
 		const branchId = branch.indexOf('/release/') >= 0 ? 0 :
 			/\/master$/.test(branch) ? 1 :
-			2; // Some unexpected branch
+				2; // Some unexpected branch
 
 		const out = cp.execSync(`git rev-list HEAD --count`);
 		const count = parseInt(out.toString());

build/lib/extensions.js

@@ -3,6 +3,14 @@
  *  Copyright (c) Microsoft Corporation. All rights reserved.
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
+var __assign = (this && this.__assign) || Object.assign || function(t) {
+    for (var s, i = 1, n = arguments.length; i < n; i++) {
+        s = arguments[i];
+        for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
+            t[p] = s[p];
+    }
+    return t;
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 var es = require("event-stream");
 var assign = require("object-assign");
@@ -14,14 +22,15 @@ var rename = require('gulp-rename');
 var util = require('gulp-util');
 var buffer = require('gulp-buffer');
 var json = require('gulp-json-editor');
+var webpack = require('webpack');
+var webpackGulp = require('webpack-stream');
 var fs = require("fs");
 var path = require("path");
 var vsce = require("vsce");
 var File = require("vinyl");
-function fromLocal(extensionPath) {
+function fromLocal(extensionPath, sourceMappingURLBase) {
     var result = es.through();
-    vsce.listFiles({ cwd: extensionPath, packageManager: vsce.PackageManager.Yarn })
-        .then(function (fileNames) {
+    vsce.listFiles({ cwd: extensionPath, packageManager: vsce.PackageManager.Yarn }).then(function (fileNames) {
         var files = fileNames
             .map(function (fileName) { return path.join(extensionPath, fileName); })
             .map(function (filePath) { return new File({
@@ -30,9 +39,58 @@ function fromLocal(extensionPath) {
             base: extensionPath,
             contents: fs.createReadStream(filePath)
         }); });
-        es.readArray(files).pipe(result);
-    })
-        .catch(function (err) { return result.emit('error', err); });
+        var filesStream = es.readArray(files);
+        // check for a webpack configuration file, then invoke webpack
+        // and merge its output with the files stream. also rewrite the package.json
+        // file to a new entry point
+        if (fs.existsSync(path.join(extensionPath, 'extension.webpack.config.js'))) {
+            var packageJsonFilter = filter('package.json', { restore: true });
+            var patchFilesStream = filesStream
+                .pipe(packageJsonFilter)
+                .pipe(buffer())
+                .pipe(json(function (data) {
+                // hardcoded entry point directory!
+                data.main = data.main.replace('/out/', /dist/);
+                return data;
+            }))
+                .pipe(packageJsonFilter.restore);
+            var webpackConfig = __assign({}, require(path.join(extensionPath, 'extension.webpack.config.js')), { mode: 'production', stats: 'errors-only' });
+            var webpackStream = webpackGulp(webpackConfig, webpack)
+                .pipe(es.through(function (data) {
+                data.stat = data.stat || {};
+                data.base = extensionPath;
+                this.emit('data', data);
+            }))
+                .pipe(es.through(function (data) {
+                // source map handling:
+                // * rewrite sourceMappingURL
+                // * save to disk so that upload-task picks this up
+                if (sourceMappingURLBase) {
+                    var contents = data.contents.toString('utf8');
+                    data.contents = Buffer.from(contents.replace(/\n\/\/# sourceMappingURL=(.*)$/gm, function (_m, g1) {
+                        return "\n//# sourceMappingURL=" + sourceMappingURLBase + "/extensions/" + path.basename(extensionPath) + "/dist/" + g1;
+                    }), 'utf8');
+                    if (/\.js\.map$/.test(data.path)) {
+                        if (!fs.existsSync(path.dirname(data.path))) {
+                            fs.mkdirSync(path.dirname(data.path));
+                        }
+                        fs.writeFileSync(data.path, data.contents);
+                    }
+                }
+                this.emit('data', data);
+            }));
+            es.merge(webpackStream, patchFilesStream)
+                // .pipe(es.through(function (data) {
+                // 	// debug
+                // 	console.log('out', data.path, data.contents.length);
+                // 	this.emit('data', data);
+                // }))
+                .pipe(result);
+        }
+        else {
+            filesStream.pipe(result);
+        }
+    }).catch(function (err) { return result.emit('error', err); });
     return result;
 }
 exports.fromLocal = fromLocal;

build/lib/extensions.ts

@@ -14,28 +14,88 @@ const rename = require('gulp-rename');
 const util = require('gulp-util');
 const buffer = require('gulp-buffer');
 const json = require('gulp-json-editor');
+const webpack = require('webpack');
+const webpackGulp = require('webpack-stream');
 import * as fs from 'fs';
 import * as path from 'path';
 import * as vsce from 'vsce';
 import * as File from 'vinyl';
 
-export function fromLocal(extensionPath: string): Stream {
-	const result = es.through();
-
-	vsce.listFiles({ cwd: extensionPath, packageManager: vsce.PackageManager.Yarn })
-		.then(fileNames => {
-			const files = fileNames
-				.map(fileName => path.join(extensionPath, fileName))
-				.map(filePath => new File({
-					path: filePath,
-					stat: fs.statSync(filePath),
-					base: extensionPath,
-					contents: fs.createReadStream(filePath) as any
-				}));
+export function fromLocal(extensionPath: string, sourceMappingURLBase?: string): Stream {
+	let result = es.through();
+
+	vsce.listFiles({ cwd: extensionPath, packageManager: vsce.PackageManager.Yarn }).then(fileNames => {
+		const files = fileNames
+			.map(fileName => path.join(extensionPath, fileName))
+			.map(filePath => new File({
+				path: filePath,
+				stat: fs.statSync(filePath),
+				base: extensionPath,
+				contents: fs.createReadStream(filePath) as any
+			}));
+
+		const filesStream = es.readArray(files);
+
+		// check for a webpack configuration file, then invoke webpack
+		// and merge its output with the files stream. also rewrite the package.json
+		// file to a new entry point
+		if (fs.existsSync(path.join(extensionPath, 'extension.webpack.config.js'))) {
+			const packageJsonFilter = filter('package.json', { restore: true });
+
+			const patchFilesStream = filesStream
+				.pipe(packageJsonFilter)
+				.pipe(buffer())
+				.pipe(json(data => {
+					// hardcoded entry point directory!
+					data.main = data.main.replace('/out/', /dist/);
+					return data;
+				}))
+				.pipe(packageJsonFilter.restore);
+
+			const webpackConfig = {
+				...require(path.join(extensionPath, 'extension.webpack.config.js')),
+				...{ mode: 'production', stats: 'errors-only' }
+			};
+			const webpackStream = webpackGulp(webpackConfig, webpack)
+				.pipe(es.through(function (data) {
+					data.stat = data.stat || {};
+					data.base = extensionPath;
+					this.emit('data', data);
+				}))
+				.pipe(es.through(function (data: File) {
+					// source map handling:
+					// * rewrite sourceMappingURL
+					// * save to disk so that upload-task picks this up
+					if (sourceMappingURLBase) {
+						const contents = (<Buffer>data.contents).toString('utf8');
+						data.contents = Buffer.from(contents.replace(/\n\/\/# sourceMappingURL=(.*)$/gm, function (_m, g1) {
+							return `\n//# sourceMappingURL=${sourceMappingURLBase}/extensions/${path.basename(extensionPath)}/dist/${g1}`;
+						}), 'utf8');
+
+						if (/\.js\.map$/.test(data.path)) {
+							if (!fs.existsSync(path.dirname(data.path))) {
+								fs.mkdirSync(path.dirname(data.path));
+							}
+							fs.writeFileSync(data.path, data.contents);
+						}
+					}
+					this.emit('data', data);
+				}))
+				;
+
+			es.merge(webpackStream, patchFilesStream)
+				// .pipe(es.through(function (data) {
+				// 	// debug
+				// 	console.log('out', data.path, data.contents.length);
+				// 	this.emit('data', data);
+				// }))
+				.pipe(result);
+
+		} else {
+			filesStream.pipe(result);
+		}
 
-			es.readArray(files).pipe(result);
-		})
-		.catch(err => result.emit('error', err));
+	}).catch(err => result.emit('error', err));
 
 	return result;
 }

extensions/emmet/.vscodeignore

 test/**
 src/**
-tsconfig.json
\ No newline at end of file
+out/**
+tsconfig.json

extensions/emmet/extension.webpack.config.js

+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+const sharedConfig = require('../shared.webpack.config');
+
+const myConfig = {
+	entry: {
+		extension: './src/extension.ts',
+	},
+	externals: {
+		'vscode': 'commonjs vscode', // ignored because it doesn't exist
+		'@emmetio/css-parser': 'commonjs @emmetio/css-parser',
+		'@emmetio/html-matcher': 'commonjs @emmetio/html-matcher',
+		'@emmetio/math-expression': 'commonjs @emmetio/math-expression',
+		'image-size': 'commonjs image-size',
+		'vscode-emmet-helper': 'commonjs vscode-emmet-helper',
+	},
+};
+
+module.exports = { ...sharedConfig(__dirname), ...myConfig };

extensions/emmet/tsconfig.json

@@ -16,4 +16,4 @@
 	"include": [
 		"src/**/*"
 	]
-}
\ No newline at end of file
+}

extensions/git/.vscodeignore

 src/**
 test/**
-out/test/**
+out/**
 tsconfig.json
-build/**
\ No newline at end of file
+build/**

extensions/git/extension.webpack.config.js

+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+const sharedConfig = require('../shared.webpack.config');
+const CopyWebpackPlugin = require('copy-webpack-plugin');
+
+const myConfig = {
+	node: {
+		__dirname: false // leave the __dirname-behaviour intact
+	},
+	entry: {
+		main: './out/main.js',
+		['askpass-main']: './out/askpass-main.js'
+	},
+	plugins: [
+		new CopyWebpackPlugin([
+			{ from: './out/*.sh', to: '[name].sh' },
+			{ from: './out/nls.*.json', to: '[name].json' }
+		])
+	],
+	externals: {
+		'vscode': 'commonjs vscode', // ignored because it doesn't exist
+		"byline": 'commonjs byline',
+		"file-type": 'commonjs file-type',
+		"iconv-lite": 'commonjs iconv-lite',
+		"jschardet": 'commonjs jschardet',
+		"vscode-extension-telemetry": 'commonjs vscode-extension-telemetry',
+		"vscode-nls": 'commonjs vscode-nls',
+		"which": 'commonjs which',
+	},
+};
+
+module.exports = { ...sharedConfig(__dirname, false), ...myConfig };

extensions/git/src/main.ts

@@ -77,7 +77,7 @@ export async function activate(context: ExtensionContext): Promise<API> {
 	commands.registerCommand('git.showOutput', () => outputChannel.show());
 	disposables.push(outputChannel);
 
-	const { name, version, aiKey } = require(context.asAbsolutePath('./package.json')) as { name: string, version: string, aiKey: string };
+	const { name, version, aiKey } = require('../package.json') as { name: string, version: string, aiKey: string };
 	const telemetryReporter = new TelemetryReporter(name, version, aiKey);
 	deactivateTasks.push(() => telemetryReporter.dispose());
 

extensions/git/tsconfig.json

@@ -16,4 +16,4 @@
 	"include": [
 		"src/**/*"
 	]
-}
\ No newline at end of file
+}

extensions/shared.webpack.config.js

+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+const path = require('path');
+
+/**
+ * Function that must be invoked with __dirname and that
+ * returns a good default configuation for extensions that
+ * want to do webpack
+ */
+module.exports = function (extensionDir, useTsLoader = true) {
+	let config = {
+		context: extensionDir,
+		mode: 'none', // this leaves the source code as close as possible to the original (when packaging we set this to 'production')
+		target: 'node', // extensions run in a node context
+		resolve: {
+			mainFields: ['main'], // prefer the main-entry of package.json files
+			extensions: [".js"]
+		},
+		module: {
+			rules: []
+		},
+		output: {
+			// all output goes into `dist`.
+			// packaging depends on that and this must always be like it
+			filename: '[name].js',
+			path: path.join(extensionDir, 'dist'),
+			libraryTarget: "commonjs",
+		},
+		// yes, really source maps
+		devtool: 'source-map'
+	};
+
+	if (useTsLoader) {
+		config.resolve.extensions = [".ts", ".js"]; // support ts-files and js-files
+		config.module.rules = [{
+			// configure TypeScript loader:
+			// * only transpile because we have a separate compilation pipeline
+			// * enable sources maps for end-to-end source maps
+			test: /\.ts$/,
+			exclude: /node_modules/,
+			use: [{
+				loader: 'ts-loader',
+				options: {
+					transpileOnly: true,
+					compilerOptions: {
+						"sourceMap": true,
+					}
+				}
+			}]
+		}];
+	}
+
+	return config;
+};

package.json

@@ -62,6 +62,7 @@
     "asar": "^0.14.0",
     "chromium-pickle-js": "^0.2.0",
     "clean-css": "3.4.6",
+    "copy-webpack-plugin": "^4.5.2",
     "coveralls": "^2.11.11",
     "cson-parser": "^1.3.3",
     "debounce": "^1.0.0",
@@ -115,6 +116,7 @@
     "rimraf": "^2.2.8",
     "sinon": "^1.17.2",
     "source-map": "^0.4.4",
+    "ts-loader": "^4.4.2",
     "tslint": "^5.9.1",
     "typescript": "2.9.2",
     "typescript-formatter": "7.1.0",
@@ -123,7 +125,10 @@
     "vinyl": "^0.4.5",
     "vinyl-fs": "^2.4.3",
     "vsce": "1.33.2",
-    "vscode-nls-dev": "3.0.7"
+    "vscode-nls-dev": "3.1.0",
+    "webpack": "^4.16.5",
+    "webpack-cli": "^3.1.0",
+    "webpack-stream": "^5.1.1"
   },
   "repository": {
     "type": "git",