Web - run smoke tests using playwright (#89918)

* playwright - initial version

* browser - use existing page and not create new context

* macOS: document how to remove the security flag

* smoke test - allow to run against server build with --build option

* do not rely on args

* fix path for windows

* smoke test - smoke 💄 and -ci option

test/automation/package.json

@@ -22,18 +22,18 @@
   "devDependencies": {
     "@types/mkdirp": "0.5.1",
     "@types/ncp": "2.0.1",
-    "@types/node": "8.0.33",
-    "@types/puppeteer": "^1.19.0",
+    "@types/debug": "4.1.5",
+    "@types/node": "^12.11.7",
     "@types/tmp": "0.1.0",
     "concurrently": "^3.5.1",
     "cpx": "^1.5.0",
-    "typescript": "2.9.2",
+    "typescript": "3.7.5",
     "watch": "^1.0.2"
   },
   "dependencies": {
     "mkdirp": "^0.5.1",
     "ncp": "^2.0.0",
-    "puppeteer": "^1.19.0",
+    "playwright": "0.10.0",
     "tmp": "0.1.0",
     "vscode-uri": "^2.0.3"
   }

test/automation/src/application.ts

@@ -126,6 +126,7 @@ export class Application {
 			extraArgs,
 			remote: this.options.remote,
 			web: this.options.web,
+			browser: this.options.browser,
 			headless: this.options.headless
 		});
 

test/automation/src/code.ts

@@ -10,7 +10,7 @@ import * as fs from 'fs';
 import * as mkdirp from 'mkdirp';
 import { tmpName } from 'tmp';
 import { IDriver, connect as connectElectronDriver, IDisposable, IElement, Thenable } from './driver';
-import { connect as connectPuppeteerDriver, launch } from './puppeteerDriver';
+import { connect as connectPlaywrightDriver, launch } from './playwrightDriver';
 import { Logger } from './logger';
 import { ncp } from 'ncp';
 import { URI } from 'vscode-uri';
@@ -101,6 +101,8 @@ export interface SpawnOptions {
 	remote?: boolean;
 	/** Run in the web */
 	web?: boolean;
+	/** A specific browser to use (requires web: true) */
+	browser?: 'chromium' | 'webkit' | 'firefox';
 	/** Run in headless mode (only applies when web is true) */
 	headless?: boolean;
 }
@@ -120,68 +122,69 @@ export async function spawn(options: SpawnOptions): Promise<Code> {
 	const outPath = codePath ? getBuildOutPath(codePath) : getDevOutPath();
 	const handle = await createDriverHandle();
 
-	const args = [
-		options.workspacePath,
-		'--skip-getting-started',
-		'--skip-release-notes',
-		'--sticky-quickopen',
-		'--disable-telemetry',
-		'--disable-updates',
-		'--disable-crash-reporter',
-		`--extensions-dir=${options.extensionsPath}`,
-		`--user-data-dir=${options.userDataDir}`,
-		'--driver', handle
-	];
-
-	const env = process.env;
-
-	if (options.remote) {
-		// Replace workspace path with URI
-		args[0] = `--${options.workspacePath.endsWith('.code-workspace') ? 'file' : 'folder'}-uri=vscode-remote://test+test/${URI.file(options.workspacePath).path}`;
-
-		if (codePath) {
-			// running against a build: copy the test resolver extension
-			const testResolverExtPath = path.join(options.extensionsPath, 'vscode-test-resolver');
-			if (!fs.existsSync(testResolverExtPath)) {
-				const orig = path.join(repoPath, 'extensions', 'vscode-test-resolver');
-				await new Promise((c, e) => ncp(orig, testResolverExtPath, err => err ? e(err) : c()));
+	let child: cp.ChildProcess | undefined;
+	let connectDriver: typeof connectElectronDriver;
+
+	if (options.web) {
+		await launch(options.userDataDir, options.workspacePath, options.codePath);
+		connectDriver = connectPlaywrightDriver.bind(connectPlaywrightDriver, !!options.headless, options.browser);
+	} else {
+		const env = process.env;
+
+		const args = [
+			options.workspacePath,
+			'--skip-getting-started',
+			'--skip-release-notes',
+			'--sticky-quickopen',
+			'--disable-telemetry',
+			'--disable-updates',
+			'--disable-crash-reporter',
+			`--extensions-dir=${options.extensionsPath}`,
+			`--user-data-dir=${options.userDataDir}`,
+			'--driver', handle
+		];
+
+		if (options.remote) {
+			// Replace workspace path with URI
+			args[0] = `--${options.workspacePath.endsWith('.code-workspace') ? 'file' : 'folder'}-uri=vscode-remote://test+test/${URI.file(options.workspacePath).path}`;
+
+			if (codePath) {
+				// running against a build: copy the test resolver extension
+				const testResolverExtPath = path.join(options.extensionsPath, 'vscode-test-resolver');
+				if (!fs.existsSync(testResolverExtPath)) {
+					const orig = path.join(repoPath, 'extensions', 'vscode-test-resolver');
+					await new Promise((c, e) => ncp(orig, testResolverExtPath, err => err ? e(err) : c()));
+				}
 			}
+			args.push('--enable-proposed-api=vscode.vscode-test-resolver');
+			const remoteDataDir = `${options.userDataDir}-server`;
+			mkdirp.sync(remoteDataDir);
+			env['TESTRESOLVER_DATA_FOLDER'] = remoteDataDir;
 		}
-		args.push('--enable-proposed-api=vscode.vscode-test-resolver');
-		const remoteDataDir = `${options.userDataDir}-server`;
-		mkdirp.sync(remoteDataDir);
-		env['TESTRESOLVER_DATA_FOLDER'] = remoteDataDir;
-	}
-
-	if (!codePath) {
-		args.unshift(repoPath);
-	}
 
-	if (options.verbose) {
-		args.push('--driver-verbose');
-	}
+		if (!codePath) {
+			args.unshift(repoPath);
+		}
 
-	if (options.log) {
-		args.push('--log', options.log);
-	}
+		if (options.verbose) {
+			args.push('--driver-verbose');
+		}
 
-	if (options.extraArgs) {
-		args.push(...options.extraArgs);
-	}
+		if (options.log) {
+			args.push('--log', options.log);
+		}
 
-	let child: cp.ChildProcess | undefined;
-	let connectDriver: typeof connectElectronDriver;
+		if (options.extraArgs) {
+			args.push(...options.extraArgs);
+		}
 
-	if (options.web) {
-		await launch(args);
-		connectDriver = connectPuppeteerDriver.bind(connectPuppeteerDriver, !!options.headless);
-	} else {
 		const spawnOptions: cp.SpawnOptions = { env };
 		child = cp.spawn(electronPath, args, spawnOptions);
 		instances.add(child);
 		child.once('exit', () => instances.delete(child!));
 		connectDriver = connectElectronDriver;
 	}
+
 	return connect(connectDriver, child, outPath, handle, options.logger);
 }
 

test/automation/src/puppeteerDriver.ts → test/automation/src/playwrightDriver.ts

@@ -3,17 +3,18 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import * as puppeteer from 'puppeteer';
+import * as playwright from 'playwright';
 import { ChildProcess, spawn } from 'child_process';
 import { join } from 'path';
 import { mkdir } from 'fs';
 import { promisify } from 'util';
 import { IDriver, IDisposable } from './driver';
+import { URI } from 'vscode-uri';
 
 const width = 1200;
 const height = 800;
 
-const vscodeToPuppeteerKey: { [key: string]: string } = {
+const vscodeToPlaywrightKey: { [key: string]: string } = {
 	cmd: 'Meta',
 	ctrl: 'Control',
 	shift: 'Shift',
@@ -26,7 +27,7 @@ const vscodeToPuppeteerKey: { [key: string]: string } = {
 	home: 'Home'
 };
 
-function buildDriver(browser: puppeteer.Browser, page: puppeteer.Page): IDriver {
+function buildDriver(browser: playwright.Browser, page: playwright.Page): IDriver {
 	const driver: IDriver = {
 		_serviceBrand: undefined,
 		getWindowIds: () => {
@@ -45,8 +46,8 @@ function buildDriver(browser: puppeteer.Browser, page: puppeteer.Page): IDriver
 				const keys = chord.split('+');
 				const keysDown: string[] = [];
 				for (let i = 0; i < keys.length; i++) {
-					if (keys[i] in vscodeToPuppeteerKey) {
-						keys[i] = vscodeToPuppeteerKey[keys[i]];
+					if (keys[i] in vscodeToPlaywrightKey) {
+						keys[i] = vscodeToPlaywrightKey[keys[i]];
 					}
 					await page.keyboard.down(keys[i]);
 					keysDown.push(keys[i]);
@@ -68,7 +69,7 @@ function buildDriver(browser: puppeteer.Browser, page: puppeteer.Page): IDriver
 			await driver.click(windowId, selector, 0, 0);
 			await timeout(100);
 		},
-		setValue: async (windowId, selector, text) => page.evaluate(`window.driver.setValue('${selector}', '${text}')`),
+		setValue: async (windowId, selector, text) => page.evaluate(`window.driver.setValue('${selector}', '${text}')`).then(undefined),
 		getTitle: (windowId) => page.evaluate(`window.driver.getTitle()`),
 		isActiveElement: (windowId, selector) => page.evaluate(`window.driver.isActiveElement('${selector}')`),
 		getElements: (windowId, selector, recursive) => page.evaluate(`window.driver.getElements('${selector}', ${recursive})`),
@@ -86,31 +87,32 @@ function timeout(ms: number): Promise<void> {
 
 // function runInDriver(call: string, args: (string | boolean)[]): Promise<any> {}
 
-let args: string[] | undefined;
 let server: ChildProcess | undefined;
 let endpoint: string | undefined;
+let workspacePath: string | undefined;
 
-export async function launch(_args: string[]): Promise<void> {
-	args = _args;
-	const agentFolder = args.filter(e => e.includes('--user-data-dir='))[0].replace('--user-data-dir=', '');
+export async function launch(userDataDir: string, _workspacePath: string, codeServerPath = process.env.VSCODE_REMOTE_SERVER_PATH): Promise<void> {
+	workspacePath = _workspacePath;
+	const agentFolder = userDataDir;
 	await promisify(mkdir)(agentFolder);
 	const env = {
 		VSCODE_AGENT_FOLDER: agentFolder,
+		VSCODE_REMOTE_SERVER_PATH: codeServerPath,
 		...process.env
 	};
 	let serverLocation: string | undefined;
-	if (process.env.VSCODE_REMOTE_SERVER_PATH) {
-		serverLocation = join(process.env.VSCODE_REMOTE_SERVER_PATH, `server.${process.platform === 'win32' ? 'cmd' : 'sh'}`);
+	if (codeServerPath) {
+		serverLocation = join(codeServerPath, `server.${process.platform === 'win32' ? 'cmd' : 'sh'}`);
 	} else {
-		serverLocation = join(args[0], `resources/server/web.${process.platform === 'win32' ? 'bat' : 'sh'}`);
+		serverLocation = join(__dirname, '..', '..', '..', `resources/server/web.${process.platform === 'win32' ? 'bat' : 'sh'}`);
 	}
 	server = spawn(
 		serverLocation,
 		['--browser', 'none', '--driver', 'web'],
 		{ env }
 	);
-	server.stderr.on('data', e => console.log('Server stderr: ' + e));
-	server.stdout.on('data', e => console.log('Server stdout: ' + e));
+	server.stderr?.on('data', e => console.log('Server stderr: ' + e));
+	server.stdout?.on('data', e => console.log('Server stdout: ' + e));
 	process.on('exit', teardown);
 	process.on('SIGINT', teardown);
 	process.on('SIGTERM', teardown);
@@ -126,7 +128,7 @@ function teardown(): void {
 
 function waitForEndpoint(): Promise<string> {
 	return new Promise<string>(r => {
-		server!.stdout.on('data', (d: Buffer) => {
+		server!.stdout?.on('data', (d: Buffer) => {
 			const matches = d.toString('ascii').match(/Web UI available at (.+)/);
 			if (matches !== null) {
 				r(matches[1]);
@@ -135,20 +137,18 @@ function waitForEndpoint(): Promise<string> {
 	});
 }
 
-export function connect(headless: boolean, outPath: string, handle: string): Promise<{ client: IDisposable, driver: IDriver }> {
+export function connect(headless: boolean, engine: 'chromium' | 'webkit' | 'firefox' = 'chromium'): Promise<{ client: IDisposable, driver: IDriver }> {
 	return new Promise(async (c) => {
-		const browser = await puppeteer.launch({
+		const browser = await playwright[engine].launch({
 			// Run in Edge dev on macOS
 			// executablePath: '/Applications/Microsoft\ Edge\ Dev.app/Contents/MacOS/Microsoft\ Edge\ Dev',
-			headless,
-			slowMo: 80,
-			args: [`--window-size=${width},${height}`]
+			headless
 		});
-		const page = (await browser.pages())[0];
+		const page = (await browser.defaultContext().pages())[0];
 		await page.setViewport({ width, height });
-		await page.goto(`${endpoint}&folder=vscode-remote://localhost:9888${args![1]}`);
+		await page.goto(`${endpoint}&folder=vscode-remote://localhost:9888${URI.file(workspacePath!).path}`);
 		const result = {
-			client: { dispose: () => teardown },
+			client: { dispose: () => teardown() },
 			driver: buildDriver(browser, page)
 		};
 		c(result);

test/automation/tsconfig.json

@@ -16,6 +16,6 @@
 	"exclude": [
 		"node_modules",
 		"out",
-		"tools",
+		"tools"
 	]
 }

test/automation/yarn.lock

@@ -2,6 +2,11 @@
 # yarn lockfile v1
 
 
+"@types/debug@4.1.5":
+  version "4.1.5"
+  resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.5.tgz#b14efa8852b7768d898906613c23f688713e02cd"
+  integrity sha512-Q1y515GcOdTHgagaVFhHnIFQ38ygs/kmxdNpvpou+raI9UO3YZcHDngBSYKQklcKlvA7iuQlmIKbzvmxcOE9CQ==
+
 "@types/mkdirp@0.5.1":
   version "0.5.1"
   resolved "https://registry.yarnpkg.com/@types/mkdirp/-/mkdirp-0.5.1.tgz#ea887cd024f691c1ca67cce20b7606b053e43b0f"
@@ -21,17 +26,10 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-12.7.1.tgz#3b5c3a26393c19b400844ac422bd0f631a94d69d"
   integrity sha512-aK9jxMypeSrhiYofWWBf/T7O+KwaiAHzM4sveCdWPn71lzUSMimRnKzhXDKfKwV1kWoBo2P1aGgaIYGLf9/ljw==
 
-"@types/node@8.0.33":
-  version "8.0.33"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-8.0.33.tgz#1126e94374014e54478092830704f6ea89df04cd"
-  integrity sha512-vmCdO8Bm1ExT+FWfC9sd9r4jwqM7o97gGy2WBshkkXbf/2nLAJQUrZfIhw27yVOtLUev6kSZc4cav/46KbDd8A==
-
-"@types/puppeteer@^1.19.0":
-  version "1.19.1"
-  resolved "https://registry.yarnpkg.com/@types/puppeteer/-/puppeteer-1.19.1.tgz#942ca62288953a0f5fbbc25c103b5f2ba28b60ab"
-  integrity sha512-ReWZvoEfMiJIA3AG+eM+nCx5GKrU2ANVYY5TC0nbpeiTCtnJbcqnmBbR8TkXMBTvLBYcuTOAELbTcuX73siDNQ==
-  dependencies:
-    "@types/node" "*"
+"@types/node@^12.11.7":
+  version "12.12.26"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-12.12.26.tgz#213e153babac0ed169d44a6d919501e68f59dea9"
+  integrity sha512-UmUm94/QZvU5xLcUlNR8hA7Ac+fGpO1EG/a8bcWVz0P0LqtxFmun9Y2bbtuckwGboWJIT70DoWq1r3hb56n3DA==
 
 "@types/tmp@0.1.0":
   version "0.1.0"
@@ -729,10 +727,10 @@ hosted-git-info@^2.1.4:
   resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.4.tgz#44119abaf4bc64692a16ace34700fed9c03e2546"
   integrity sha512-pzXIvANXEFrc5oFFXRMkbLPQ2rXRoDERwDLyrcUxGhaZhgP54BBSl9Oheh7Vv0T090cszWBxPjkQQ5Sq1PbBRQ==
 
-https-proxy-agent@^2.2.1:
-  version "2.2.3"
-  resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-2.2.3.tgz#fb6cd98ed5b9c35056b5a73cd01a8a721d7193d1"
-  integrity sha512-Ytgnz23gm2DVftnzqRRz2dOXZbGd2uiajSw/95bPp6v53zPRspQjLm/AfBgqbJ2qfeRXWIOMVLpp86+/5yX39Q==
+https-proxy-agent@^3.0.0:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-3.0.1.tgz#b8c286433e87602311b01c8ea34413d856a4af81"
+  integrity sha512-+ML2Rbh6DAuee7d07tYGEKOEi2voWPUGan+ExdPbPW6Z3svq+JCqr0v8WmKPOkz1vOVykPCBSuobe7G8GJUtVg==
   dependencies:
     agent-base "^4.3.0"
     debug "^3.1.0"
@@ -938,6 +936,11 @@ isobject@^3.0.0, isobject@^3.0.1:
   resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df"
   integrity sha1-TkMekrEalzFjaqH5yNHMvP2reN8=
 
+jpeg-js@^0.3.6:
+  version "0.3.6"
+  resolved "https://registry.yarnpkg.com/jpeg-js/-/jpeg-js-0.3.6.tgz#c40382aac9506e7d1f2d856eb02f6c7b2a98b37c"
+  integrity sha512-MUj2XlMB8kpe+8DJUGH/3UJm4XpI8XEgZQ+CiHDeyrGoKPdW/8FJv6ku+3UiYm5Fz3CWaL+iXmD8Q4Ap6aC1Jw==
+
 json-parse-better-errors@^1.0.1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz#bb867cfb3450e69107c131d1c514bab3dc8bcaa9"
@@ -1320,6 +1323,35 @@ pify@^3.0.0:
   resolved "https://registry.yarnpkg.com/pify/-/pify-3.0.0.tgz#e5a4acd2c101fdf3d9a4d07f0dbc4db49dd28176"
   integrity sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=
 
+playwright-core@=0.10.0:
+  version "0.10.0"
+  resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-0.10.0.tgz#86699c9cc3e613d733e6635a54aceea1993013d5"
+  integrity sha512-yernA6yrrBhmb8M5eO6GZsJOrBKWOZszlu65Luz8LP7ryaDExN1sE9XjQBNbiwJ5Gfs8cehtAO7GfTDJt+Z2cQ==
+  dependencies:
+    debug "^4.1.0"
+    extract-zip "^1.6.6"
+    https-proxy-agent "^3.0.0"
+    jpeg-js "^0.3.6"
+    mime "^2.0.3"
+    pngjs "^3.4.0"
+    progress "^2.0.3"
+    proxy-from-env "^1.0.0"
+    rimraf "^2.6.1"
+    uuid "^3.4.0"
+    ws "^6.1.0"
+
+playwright@0.10.0:
+  version "0.10.0"
+  resolved "https://registry.yarnpkg.com/playwright/-/playwright-0.10.0.tgz#d37f7e42e0e868dcc4ec35cb0a8dbc6248457642"
+  integrity sha512-f3VRME/PIO5NbcWnlCDfXwPC0DAZJ7ETkcAdE+sensLCOkfDtLh97E71ZuxNCaPYsUA6FIPi5syD8pHJW/4hQQ==
+  dependencies:
+    playwright-core "=0.10.0"
+
+pngjs@^3.4.0:
+  version "3.4.0"
+  resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-3.4.0.tgz#99ca7d725965fb655814eaf65f38f12bbdbf555f"
+  integrity sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w==
+
 posix-character-classes@^0.1.0:
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/posix-character-classes/-/posix-character-classes-0.1.1.tgz#01eac0fe3b5af71a2a6c02feabb8c1fef7e00eab"
@@ -1335,7 +1367,7 @@ process-nextick-args@~2.0.0:
   resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2"
   integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==
 
-progress@^2.0.1:
+progress@^2.0.3:
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8"
   integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==
@@ -1345,20 +1377,6 @@ proxy-from-env@^1.0.0:
   resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.0.0.tgz#33c50398f70ea7eb96d21f7b817630a55791c7ee"
   integrity sha1-M8UDmPcOp+uW0h97gXYwpVeRx+4=
 
-puppeteer@^1.19.0:
-  version "1.19.0"
-  resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-1.19.0.tgz#e3b7b448c2c97933517078d7a2c53687361bebea"
-  integrity sha512-2S6E6ygpoqcECaagDbBopoSOPDv0pAZvTbnBgUY+6hq0/XDFDOLEMNlHF/SKJlzcaZ9ckiKjKDuueWI3FN/WXw==
-  dependencies:
-    debug "^4.1.0"
-    extract-zip "^1.6.6"
-    https-proxy-agent "^2.2.1"
-    mime "^2.0.3"
-    progress "^2.0.1"
-    proxy-from-env "^1.0.0"
-    rimraf "^2.6.1"
-    ws "^6.1.0"
-
 randomatic@^3.0.0:
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/randomatic/-/randomatic-3.1.1.tgz#b776efc59375984e36c537b2f51a1f0aff0da1ed"
@@ -1751,10 +1769,10 @@ typedarray@^0.0.6:
   resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777"
   integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=
 
-typescript@2.9.2:
-  version "2.9.2"
-  resolved "https://registry.yarnpkg.com/typescript/-/typescript-2.9.2.tgz#1cbf61d05d6b96269244eb6a3bce4bd914e0f00c"
-  integrity sha512-Gr4p6nFNaoufRIY4NMdpQRNmgxVIGMs4Fcu/ujdYk3nAZqk7supzBE9idmvfZIlH/Cuj//dvi+019qEue9lV0w==
+typescript@3.7.5:
+  version "3.7.5"
+  resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.7.5.tgz#0692e21f65fd4108b9330238aac11dd2e177a1ae"
+  integrity sha512-/P5lkRXkWHNAbcJIiHPfRoKqyd7bsyCma1hZNUGfn20qm64T6ZBlrzprymeu918H+mB/0rIg2gGK/BXkhhYgBw==
 
 union-value@^1.0.0:
   version "1.0.1"
@@ -1789,6 +1807,11 @@ util-deprecate@~1.0.1:
   resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
   integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=
 
+uuid@^3.4.0:
+  version "3.4.0"
+  resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee"
+  integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==
+
 validate-npm-package-license@^3.0.1:
   version "3.0.4"
   resolved "https://registry.yarnpkg.com/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz#fc91f6b9c7ba15c857f4cb2c5defeec39d4f410a"

test/smoke/README.md

@@ -7,15 +7,22 @@ Make sure you are on **Node v10.x**.
 ```bash
 # Install Dependencies and Compile
 yarn --cwd test/smoke
+yarn --cwd test/automation
 
-# Dev
+# Dev (Electron)
 yarn smoketest
 
-# Build
-yarn smoketest --build PATH_TO_NEW_BUILD_PARENT_FOLDER --stable-build PATH_TO_LAST_STABLE_BUILD_PARENT_FOLDER
+# Dev (Web)
+yarn smoketest --web --browser <chromium|firefox|webkit>
 
-# Remote
-yarn smoketest --build PATH_TO_NEW_BUILD_PARENT_FOLDER --remote
+# Build (Electron)
+yarn smoketest --build <path latest built version> --stable-build <path to previous stable version>
+
+# Build (Web - read instructions below)
+yarn smoketest --build <path to web server folder> --web --browser <chromium|firefox|webkit>
+
+# Remote (Electron)
+yarn smoketest --build <path latest built version> --remote
 ```
 
 ### Run for a release
@@ -27,18 +34,33 @@ git checkout release/1.22
 yarn --cwd test/smoke
 ```
 
+#### Electron
+
 In addition to the new build to be released you will need the previous stable build so that the smoketest can test the data migration.
 The recommended way to make these builds available for the smoketest is by downloading their archive version (\*.zip) and extracting
 them into two folders. Pass the folder paths to the smoketest as follows:
 
 ```bash
-yarn smoketest --build PATH_TO_NEW_RELEASE_PARENT_FOLDER --stable-build PATH_TO_LAST_STABLE_RELEASE_PARENT_FOLDER
+yarn smoketest --build <path latest built version> --stable-build <path to previous stable version>
+```
+
+#### Web
+
+**macOS**: if you have downloaded the server with web bits, make sure to run the following command before unzipping it to avoid security issues on startup:
+
+```bash
+xattr -d com.apple.quarantine <path to server with web folder zip>
 ```
 
+There is no support for testing an old version to a new one yet, so simply configure the `--build` command line argument to point to
+the web server folder which includes the web client bits (e.g. `vscode-server-darwin-web` for macOS).
+
+**Note**: make sure to point to the server that includes the client bits!
+
 ### Debug
 
 - `--verbose` logs all the low level driver calls made to Code;
-- `-f PATTERN` filters the tests to be run. You can also use pretty much any mocha argument;
+- `-f PATTERN` (alias `-g PATTERN`) filters the tests to be run. You can also use pretty much any mocha argument;
 - `--screenshots SCREENSHOT_DIR` captures screenshots when tests fail.
 
 ### Develop

test/smoke/package.json

@@ -27,7 +27,7 @@
     "rimraf": "^2.6.1",
     "strip-json-comments": "^2.0.1",
     "tmp": "0.0.33",
-    "typescript": "2.9.2",
+    "typescript": "3.7.5",
     "watch": "^1.0.2"
   }
 }

test/smoke/src/areas/workbench/data-migration.test.ts

@@ -8,7 +8,6 @@ import { join } from 'path';
 
 export function setup(stableCodePath: string, testDataPath: string) {
 
-
 	describe('Data Migration: This test MUST run before releasing by providing the --stable-build command line argument', () => {
 		it(`verifies opened editors are restored`, async function () {
 			if (!stableCodePath) {

test/smoke/src/main.ts

@@ -47,6 +47,7 @@ process.once('exit', () => rimraf.sync(testDataPath));
 const [, , ...args] = process.argv;
 const opts = minimist(args, {
 	string: [
+		'browser',
 		'build',
 		'stable-build',
 		'wait-time',
@@ -58,7 +59,8 @@ const opts = minimist(args, {
 		'verbose',
 		'remote',
 		'web',
-		'headless'
+		'headless',
+		'ci'
 	],
 	default: {
 		verbose: false
@@ -82,42 +84,46 @@ function fail(errorMessage): void {
 
 const repoPath = path.join(__dirname, '..', '..', '..');
 
-function getDevElectronPath(): string {
-	const buildPath = path.join(repoPath, '.build');
-	const product = require(path.join(repoPath, 'product.json'));
-
-	switch (process.platform) {
-		case 'darwin':
-			return path.join(buildPath, 'electron', `${product.nameLong}.app`, 'Contents', 'MacOS', 'Electron');
-		case 'linux':
-			return path.join(buildPath, 'electron', `${product.applicationName}`);
-		case 'win32':
-			return path.join(buildPath, 'electron', `${product.nameShort}.exe`);
-		default:
-			throw new Error('Unsupported platform.');
-	}
-}
+let quality: Quality;
 
-function getBuildElectronPath(root: string): string {
-	switch (process.platform) {
-		case 'darwin':
-			return path.join(root, 'Contents', 'MacOS', 'Electron');
-		case 'linux': {
-			const product = require(path.join(root, 'resources', 'app', 'product.json'));
-			return path.join(root, product.applicationName);
-		}
-		case 'win32': {
-			const product = require(path.join(root, 'resources', 'app', 'product.json'));
-			return path.join(root, `${product.nameShort}.exe`);
+//
+// #### Electron Smoke Tests ####
+//
+if (!opts.web) {
+
+	function getDevElectronPath(): string {
+		const buildPath = path.join(repoPath, '.build');
+		const product = require(path.join(repoPath, 'product.json'));
+
+		switch (process.platform) {
+			case 'darwin':
+				return path.join(buildPath, 'electron', `${product.nameLong}.app`, 'Contents', 'MacOS', 'Electron');
+			case 'linux':
+				return path.join(buildPath, 'electron', `${product.applicationName}`);
+			case 'win32':
+				return path.join(buildPath, 'electron', `${product.nameShort}.exe`);
+			default:
+				throw new Error('Unsupported platform.');
 		}
-		default:
-			throw new Error('Unsupported platform.');
 	}
-}
 
-let quality: Quality;
+	function getBuildElectronPath(root: string): string {
+		switch (process.platform) {
+			case 'darwin':
+				return path.join(root, 'Contents', 'MacOS', 'Electron');
+			case 'linux': {
+				const product = require(path.join(root, 'resources', 'app', 'product.json'));
+				return path.join(root, product.applicationName);
+			}
+			case 'win32': {
+				const product = require(path.join(root, 'resources', 'app', 'product.json'));
+				return path.join(root, `${product.nameShort}.exe`);
+			}
+			default:
+				throw new Error('Unsupported platform.');
+		}
+	}
 
-if (!opts.web) {
 	let testCodePath = opts.build;
 	let stableCodePath = opts['stable-build'];
 	let electronPath: string;
@@ -152,8 +158,13 @@ if (!opts.web) {
 	} else {
 		quality = Quality.Stable;
 	}
-} else {
-	let testCodeServerPath = process.env.VSCODE_REMOTE_SERVER_PATH;
+}
+
+//
+// #### Web Smoke Tests ####
+//
+else {
+	const testCodeServerPath = opts.build || process.env.VSCODE_REMOTE_SERVER_PATH;
 
 	if (typeof testCodeServerPath === 'string' && !fs.existsSync(testCodeServerPath)) {
 		fail(`Can't find Code server at ${testCodeServerPath}.`);
@@ -236,13 +247,13 @@ function createOptions(): ApplicationOptions {
 		screenshotsPath,
 		remote: opts.remote,
 		web: opts.web,
+		browser: opts.browser,
 		headless: opts.headless
 	};
 }
 
 before(async function () {
-	// allow two minutes for setup
-	this.timeout(2 * 60 * 1000);
+	this.timeout(2 * 60 * 1000); // allow two minutes for setup
 	await setup();
 	this.defaultOptions = createOptions();
 });
@@ -259,11 +270,7 @@ after(async function () {
 	await new Promise((c, e) => rimraf(testDataPath, { maxBusyTries: 10 }, err => err ? e(err) : c()));
 });
 
-if (!opts.web) {
-	setupDataMigrationTests(opts['stable-build'], testDataPath);
-}
-
-describe('Running Code', () => {
+describe(`VSCode Smoke Tests (${opts.web ? 'Web' : 'Electron'})`, () => {
 	before(async function () {
 		const app = new Application(this.defaultOptions);
 		await app!.start(opts.web ? false : undefined);
@@ -295,19 +302,25 @@ describe('Running Code', () => {
 		});
 	}
 
-	if (!opts.web) { setupDataLossTests(); }
-	setupDataExplorerTests();
-	if (!opts.web) { setupDataPreferencesTests(); }
-	setupDataSearchTests();
-	setupDataCSSTests();
-	setupDataEditorTests();
-	setupDataStatusbarTests(!!opts.web);
-	setupDataExtensionTests();
-	setupTerminalTests();
-	if (!opts.web) { setupDataMultirootTests(); }
-	setupDataLocalizationTests();
-});
+	// CI only tests (must be reliable)
+	if (opts.ci) {
+		// TODO@Ben figure out tests that can run continously and reliably
+	}
 
-if (!opts.web) {
-	setupLaunchTests();
-}
+	// Non-CI execution (all tests)
+	else {
+		if (!opts.web) { setupDataMigrationTests(opts['stable-build'], testDataPath); }
+		if (!opts.web) { setupDataLossTests(); }
+		setupDataExplorerTests();
+		if (!opts.web) { setupDataPreferencesTests(); }
+		setupDataSearchTests();
+		setupDataCSSTests();
+		setupDataEditorTests();
+		setupDataStatusbarTests(!!opts.web);
+		setupDataExtensionTests();
+		setupTerminalTests();
+		if (!opts.web) { setupDataMultirootTests(); }
+		setupDataLocalizationTests();
+		if (!opts.web) { setupLaunchTests(); }
+	}
+});

test/smoke/test/index.js

@@ -11,16 +11,14 @@ const suite = 'Smoke Tests';
 
 const [, , ...args] = process.argv;
 const opts = minimist(args, {
-	string: [
-		'f'
-	]
+	string: ['f', 'g']
 });
 
 const options = {
-	useColors: true,
+	color: true,
 	timeout: 60000,
 	slow: 30000,
-	grep: opts['f']
+	grep: opts['f'] || opts['g']
 };
 
 if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {

test/smoke/yarn.lock

@@ -2122,10 +2122,10 @@ tree-kill@^1.1.0:
   resolved "https://registry.yarnpkg.com/tree-kill/-/tree-kill-1.2.0.tgz#5846786237b4239014f05db156b643212d4c6f36"
   integrity sha512-DlX6dR0lOIRDFxI0mjL9IYg6OTncLm/Zt+JiBhE5OlFcAR8yc9S7FFXU9so0oda47frdM/JFsk7UjNt9vscKcg==
 
-typescript@2.9.2:
-  version "2.9.2"
-  resolved "https://registry.yarnpkg.com/typescript/-/typescript-2.9.2.tgz#1cbf61d05d6b96269244eb6a3bce4bd914e0f00c"
-  integrity sha512-Gr4p6nFNaoufRIY4NMdpQRNmgxVIGMs4Fcu/ujdYk3nAZqk7supzBE9idmvfZIlH/Cuj//dvi+019qEue9lV0w==
+typescript@3.7.5:
+  version "3.7.5"
+  resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.7.5.tgz#0692e21f65fd4108b9330238aac11dd2e177a1ae"
+  integrity sha512-/P5lkRXkWHNAbcJIiHPfRoKqyd7bsyCma1hZNUGfn20qm64T6ZBlrzprymeu918H+mB/0rIg2gGK/BXkhhYgBw==
 
 union-value@^1.0.0:
   version "1.0.1"