Merge branch 'smoketest'

build/npm/postinstall.js

@@ -58,4 +58,5 @@ function npmInstallBuildDependencies() {
 }
 
 npmInstall(`build`); // node modules required for build
+npmInstall('test/smoke'); // node modules required for smoketest
 npmInstallBuildDependencies(); // node modules for watching, specific to host node version, not electron
\ No newline at end of file

build/tfs/darwin/smoketest.sh

@@ -24,5 +24,5 @@ step "Build minified & upload source maps" \
 step "Run smoke test" \
 	pushd test/smoke
 	npm install
-	npm test -- --latest "$AGENT_BUILDDIRECTORY/VSCode-darwin/Visual Studio Code - Insiders.app/Contents/MacOS/Electron"
+	npm run smoketest -- "$AGENT_BUILDDIRECTORY/VSCode-darwin/Visual Studio Code - Insiders.app/Contents/MacOS/Electron"
 	popd
\ No newline at end of file

build/tfs/linux/smoketest.sh

@@ -36,7 +36,7 @@ function configureEnvironment {
 function runTest {
 	pushd test/smoke
 	npm install
-	sudo -u testuser -H xvfb-run -a -s "-screen 0 1024x768x8" npm test -- --latest "$AGENT_BUILDDIRECTORY/VSCode-linux-ia32/code-insiders"
+	sudo -u testuser -H xvfb-run -a -s "-screen 0 1024x768x8" npm run smoketest -- "$AGENT_BUILDDIRECTORY/VSCode-linux-ia32/code-insiders"
 	popd
 }
 

build/tfs/win32/smoketest.ps1

@@ -40,7 +40,7 @@ step "Build minified" {
 step "Run smoke test" {
 	exec { & Push-Location test\smoke }
 	exec { & npm install }
-	exec { & npm test -- --latest "$env:AGENT_BUILDDIRECTORY\VSCode-win32-$global:arch\Code - Insiders.exe" }
+	exec { & npm run smoketest -- "$env:AGENT_BUILDDIRECTORY\VSCode-win32-$global:arch\Code - Insiders.exe" }
 	exec { & Pop-Location }
 }
 

package.json

@@ -19,7 +19,8 @@
     "precommit": "node build/gulpfile.hygiene.js",
     "gulp": "gulp --max_old_space_size=4096",
     "7z": "7z",
-    "update-grammars": "node build/npm/update-all-grammars.js"
+    "update-grammars": "node build/npm/update-all-grammars.js",
+    "smoketest": "node test/smoke/out/main.js"
   },
   "dependencies": {
     "applicationinsights": "0.17.1",

test/smoke/README.md

 # VS Code Smoke Testing
 
-- Run `npm install`
-- Start the tests: `npm test -- --latest "path/to/binary"`.
+```
+npm rum smoketest -- "path/to/code"
+```
 
-If you want to include 'Data Migration' area tests use  `npm test -- --latest path/to/binary --stable path/to/currentStable` respectively.
+If you want to include 'Data Migration' area tests use:
+
+```
+npm run smoketest -- "path/to/code" "path/to/codeStable"
+```
 
 Detailed prerequisites and running steps are described [in our smoke test wiki](https://github.com/Microsoft/vscode/wiki/Smoke-Test#automated-smoke-test).
 
@@ -50,15 +55,7 @@ To add new test, `./test/${area}.ts` should be updated. The same instruction-sty
 Almost on every automated test action it captures a screenshot. These help to determine an issue, if smoke test fails. The normal workflow is that you understand what code is doing and then try to match it up with screenshots obtained from the test.
 
 # Running "Out of Sources"
-If you did a fix in VS Code that you need in order for the smoke test to succeed, here is how you can run the smoke test against the sources of VS Code:
-* Set related environment variables in the console:
-  * `export NODE_ENV=development`
-  * `export VSCODE_DEV=1`
-  * `export VSCODE_CLI=1`
+
+```
+npm run smoketest
+```
\ No newline at end of file
-* open `application.ts`
-  * pass in the vscode folder as argument to the application
-  * e.g. instead of `args: args` type `args: ['/Users/bpasero/Development/vscode', ...args]`
-* `cd test/smoke`
-* `npm install`
-* `npm test -- --latest <path to electron>`
-  * e.g. on macOS: `npm test -- --latest <path to vscode>/.build/electron/Code\ -\ OSS.app/Contents/MacOS/Electron`
\ No newline at end of file

test/smoke/package.json

@@ -3,24 +3,22 @@
   "version": "0.1.0",
   "main": "./src/main.js",
   "scripts": {
-    "compile": "tsc",
-    "pretest": "tsc",
-    "test": "node out/main.js"
+    "postinstall": "tsc",
+    "watch": "tsc --watch"
   },
   "devDependencies": {
-    "@types/mocha": "^2.2.41",
-    "@types/node": "^6.0.70",
-    "@types/webdriverio": "^4.6.1",
     "@types/electron": "~1.4.37",
-    "@types/rimraf": "^0.0.28",
     "@types/htmlparser2": "^3.7.29",
+    "@types/mkdirp": "^0.5.1",
+    "@types/mocha": "^2.2.41",
+    "@types/node": "^8.0.26",
+    "@types/rimraf": "^0.0.28",
+    "@types/webdriverio": "^4.6.1",
+    "htmlparser2": "^3.9.2",
     "mocha": "^3.2.0",
-    "spectron": "~3.6.4",
-    "typescript": "^2.2.2",
     "rimraf": "^2.6.1",
-    "commander": "^2.9.0",
-    "simple-git": "^1.73.0",
+    "spectron": "~3.6.4",
     "strip-json-comments": "^2.0.1",
-    "htmlparser2": "^3.9.2"
+    "typescript": "^2.2.2"
   }
 }
\ No newline at end of file

test/smoke/src/main.ts

@@ -3,220 +3,132 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-const fs = require('fs');
-const https = require('https');
-const program = require('commander');
-const git = require('simple-git')();
-const child_process = require('child_process');
-const path = require('path');
-const mkdirp = require('mkdirp');
-
-const testDataPath = path.join(process.cwd(), 'test_data');
-const codeWorkspacePath = path.join(testDataPath, 'smoketest.code-workspace');
+import * as fs from 'fs';
+import * as https from 'https';
+import * as cp from 'child_process';
+import * as path from 'path';
+import * as mkdirp from 'mkdirp';
+
+const testDataPath = path.join(__dirname, '..', 'test_data');
+const workspacePath = path.join(testDataPath, 'smoketest.code-workspace');
 const testRepoUrl = 'https://github.com/Microsoft/vscode-smoketest-express';
 const testRepoLocalDir = path.join(testDataPath, 'vscode-smoketest-express');
-const keybindingsUrl = 'https://raw.githubusercontent.com/Microsoft/vscode-docs/master/scripts/keybindings';
-
 mkdirp.sync(testDataPath);
 
-program
-	.option('-l, --latest <file path>', 'path to the latest VS Code to test')
-	.option('-s, --stable [file path]', 'path to the stable VS Code to be used in data migration tests');
-
-program.on('--help', () => {
-	console.log('  Examples:');
-	console.log('');
-	console.log('    $ npm test -- --latest path/to/binary');
-	console.log('    $ npm test -- -l path/to/binary');
-	console.log('');
-	console.log('    $ npm test -- --latest path/to/latest/binary --stable path/to/stable/binary');
-	console.log('    $ npm test -- -l path/to/latest/binary -s path/to/stable/binary');
-	console.log('');
-});
-program.parse(process.argv);
-
-if (!program.latest) {
-	fail('You must specify the binary to run the smoke test against');
-}
-if (!binaryExists(program.latest) || (program.stable && !binaryExists(program.stable))) {
-	fail('The file path to electron binary does not exist or permissions do not allow to execute it. Please check the path provided.');
+function fail(errorMessage): void {
+	console.error(errorMessage);
+	process.exit(1);
 }
+
 if (parseInt(process.version.substr(1)) < 6) {
 	fail('Please update your Node version to greater than 6 to run the smoke test.');
 }
 
-// Setting up environment variables
-process.env.VSCODE_LATEST_PATH = program.latest;
-if (program.stable) {
-	process.env.VSCODE_STABLE_PATH = program.stable;
-}
-process.env.SMOKETEST_REPO = testRepoLocalDir;
-if (program.latest && (program.latest.indexOf('Code - Insiders') /* macOS/Windows */ || program.latest.indexOf('code-insiders') /* Linux */) >= 0) {
-	process.env.VSCODE_EDITION = 'insiders';
+const repoPath = path.join(__dirname, '..', '..', '..');
+
+function getDevElectronPath(): string {
+	const buildPath = path.join(repoPath, '.build');
+	const product = require(path.join(repoPath, 'product.json'));
+
+	switch (process.platform) {
+		case 'darwin':
+			return path.join(buildPath, `${product.nameLong}.app`, 'Contents', 'MacOS', 'Electron');
+		case 'linux':
+			return path.join(buildPath, 'electron', `${product.applicationName}`);
+		case 'win32':
+			return path.join(buildPath, 'electron', `${product.nameShort}.exe`);
+		default:
+			throw new Error('Unsupported platform.');
+	}
 }
-process.env.VSCODE_WORKSPACE_PATH = codeWorkspacePath;
 
-// Setting up 'vscode-smoketest-express' project
-let os = process.platform.toString();
-if (os === 'darwin') {
-	os = 'osx';
+let [, , testCodePath, stableCodePath] = process.argv;
+
+if (testCodePath) {
+	process.env.VSCODE_PATH = testCodePath;
+
+	if (stableCodePath) {
+		process.env.VSCODE_STABLE_PATH = stableCodePath;
+	}
+} else {
+	testCodePath = getDevElectronPath();
+	process.env.VSCODE_PATH = testCodePath;
+	process.env.VSCODE_REPOSITORY = repoPath;
+	process.env.VSCODE_DEV = '1';
+	process.env.VSCODE_CLI = '1';
 }
-else if (os === 'win32') {
-	os = 'win';
+
+if (!fs.existsSync(testCodePath)) {
+	fail(`Can't find Code at ${testCodePath}.`);
 }
 
-main().catch(err => console.error(err));
+process.env.SMOKETEST_REPO = testRepoLocalDir;
+process.env.VSCODE_WORKSPACE_PATH = workspacePath;
 
-async function main(): Promise<void> {
-	await getKeybindings(`${keybindingsUrl}/doc.keybindings.${os}.json`, path.join(testDataPath, 'keybindings.json'));
-
-	const workspace = {
-		id: (Date.now() + Math.round(Math.random() * 1000)).toString(),
-		folders: [
-			toUri(path.join(testRepoLocalDir, 'public')),
-			toUri(path.join(testRepoLocalDir, 'routes')),
-			toUri(path.join(testRepoLocalDir, 'views'))
-		]
-	};
-
-	await createWorkspaceFile(codeWorkspacePath, workspace);
-	await cleanOrClone(testRepoUrl, testRepoLocalDir);
-	await execute('npm install', testRepoLocalDir);
-	await runTests();
+if ((testCodePath.indexOf('Code - Insiders') /* macOS/Windows */ || testCodePath.indexOf('code-insiders') /* Linux */) >= 0) {
+	process.env.VSCODE_EDITION = 'insiders';
 }
 
-function fail(errorMessage): void {
-	console.error(errorMessage);
-	process.exit(1);
+function getKeybindingPlatform(): string {
+	switch (process.platform) {
+		case 'darwin': return 'osx';
+		case 'win32': return 'win';
+		default: return process.platform;
+	}
 }
 
 function toUri(path: string): string {
-	if (os === 'win') {
+	if (process.platform === 'win32') {
 		return `file:///${path.replace(/\\/g, '/')}`;
 	}
 
 	return `file://${path}`;
 }
 
-function runTests(): void {
-	console.log('Running tests...');
-	var proc = child_process.spawn(process.execPath, [
-		'out/mocha-runner.js'
-	]);
-	proc.stdout.on('data', data => {
-		console.log(data.toString());
-	});
-	proc.stderr.on('data', data => {
-		var date = new Date().toLocaleString();
-		fs.appendFile(path.join(testDataPath, 'errors.log'), `${date}: ${data.toString()}`, (err) => {
-			if (err) {
-				throw new Error(`Could not write stderr to errors.log with the following error: ${err}`);
-			};
-		});
-	});
-	proc.on('exit', (code) => {
-		process.exit(code);
+async function main(): Promise<void> {
+	const keybindingsUrl = `https://raw.githubusercontent.com/Microsoft/vscode-docs/master/scripts/keybindings/doc.keybindings.${getKeybindingPlatform()}.json`;
+	console.log(`Fetching keybindings from ${keybindingsUrl}...`);
+
+	await new Promise((c, e) => {
+		https.get(keybindingsUrl, res => {
+			const output = fs.createWriteStream(path.join(testDataPath, 'keybindings.json'));
+			res.on('error', e);
+			output.on('error', e);
+			output.on('close', c);
+			res.pipe(output);
+		}).on('error', e);
 	});
-}
 
-async function cleanOrClone(repo: string, dir: string): Promise<any> {
-	console.log('Cleaning or cloning test project repository...');
+	if (!fs.existsSync(workspacePath)) {
+		console.log('Creating workspace file...');
+		const workspace = {
+			id: (Date.now() + Math.round(Math.random() * 1000)).toString(),
+			folders: [
+				toUri(path.join(testRepoLocalDir, 'public')),
+				toUri(path.join(testRepoLocalDir, 'routes')),
+				toUri(path.join(testRepoLocalDir, 'views'))
+			]
+		};
+
+		fs.writeFileSync(workspacePath, JSON.stringify(workspace, null, '\t'));
+	}
 
-	if (!folderExists(dir)) {
-		await gitClone(repo, dir);
+	if (!fs.existsSync(testRepoLocalDir)) {
+		console.log('Cloning test project repository...');
+		cp.spawnSync('git', ['clone', testRepoUrl, testRepoLocalDir]);
 	} else {
-		git.cwd(dir);
-		await new Promise((c, e) => git.fetch(err => err ? e(err) : c()));
-		await gitResetAndClean();
+		console.log('Cleaning test project repository...');
+		cp.spawnSync('git', ['fetch'], { cwd: testRepoLocalDir });
+		cp.spawnSync('git', ['reset', '--hard', 'FETCH_HEAD'], { cwd: testRepoLocalDir });
+		cp.spawnSync('git', ['clean', '-xdf'], { cwd: testRepoLocalDir });
 	}
-}
-
-function gitClone(repo: string, dir: string): Promise<any> {
-	return new Promise((res, rej) => {
-		git.clone(repo, dir, () => {
-			console.log('Test repository successfully cloned.');
-			res();
-		});
-	});
-}
 
-async function gitResetAndClean(): Promise<any> {
-	await new Promise((c, e) => git.reset(['FETCH_HEAD', '--hard'], err => err ? e(err) : c()));
-	await new Promise((c, e) => git.clean('f', ['-d'], err => err ? e(err) : c()));
-	console.log('Test project was successfully reset to initial state.');
-}
+	console.log('Running npm install...');
+	cp.execSync('npm install', { cwd: testRepoLocalDir, stdio: 'inherit' });
 
-function execute(cmd: string, dir: string): Promise<any> {
-	return new Promise((res, rej) => {
-		console.log(`Running ${cmd}...`);
-		child_process.exec(cmd, { cwd: dir, stdio: [0, 1, 2] }, (error, stdout, stderr) => {
-			if (error) {
-				rej(error);
-			}
-			if (stderr) {
-				console.error(stderr);
-			}
-			console.log(stdout);
-			res();
-		});
-	});
-}
-
-function getKeybindings(url: string, location: string): Promise<any> {
-	console.log(`Fetching keybindings from ${url}...`);
-	return new Promise((resolve, reject) => {
-		https.get(url, (res) => {
-			if (res.statusCode !== 200) {
-				reject(`Failed to obtain key bindings with response code: ${res.statusCode}`);
-			}
-
-			var buffer: Buffer[] = [];
-			res.on('data', (chunk) => buffer.push(chunk));
-			res.on('end', () => {
-				fs.writeFile(location, Buffer.concat(buffer), 'utf8', () => {
-					console.log('Keybindings were successfully fetched.');
-					resolve();
-				});
-			});
-		}).on('error', (e) => {
-			reject(`Failed to obtain key bindings with an error: ${e}`);
-		});
-	});
-}
-
-function createWorkspaceFile(path: string, workspace: any): Promise<any> {
-	console.log(`Creating workspace file at ${path}...`);
-	return new Promise((resolve, reject) => {
-		fs.exists(path, exists => {
-			if (exists) {
-				return resolve();
-			}
-
-			fs.writeFile(path, JSON.stringify(workspace, null, '\t'), error => {
-				if (error) {
-					reject(error);
-				} else {
-					resolve();
-				}
-			});
-		});
-	});
-}
-
-function folderExists(folder: string): boolean {
-	try {
-		fs.accessSync(folder, 'rw');
-		return true;
-	} catch (e) {
-		return false;
-	}
+	console.log('Running tests...');
+	const mocha = cp.spawnSync(process.execPath, ['out/mocha-runner.js'], { cwd: path.join(__dirname, '..'), stdio: 'inherit' });
+	process.exit(mocha.status);
 }
 
-function binaryExists(filePath: string): boolean {
-	try {
+main().catch(fail);
\ No newline at end of file
-		fs.accessSync(filePath, 'x');
-		return true;
-	} catch (e) {
-		return false;
-	}
-}
\ No newline at end of file

test/smoke/src/mocha-runner.ts

@@ -4,13 +4,14 @@
  *--------------------------------------------------------------------------------------------*/
 
 const MochaTest = require('mocha');
+const path = require('path');
 
 const mochaTest = new MochaTest({
 	timeout: 60000,
 	slow: 10000,
 	useColors: true
 });
-mochaTest.addFile(require('path').join(process.cwd(), 'out/test.js'));
+mochaTest.addFile(path.join(__dirname, 'test.js'));
 mochaTest.run((failures) => {
 	process.exit(failures);
 });
\ No newline at end of file

test/smoke/src/spectron/application.ts

@@ -9,10 +9,10 @@ import { Screenshot } from '../helpers/screenshot';
 var fs = require('fs');
 var path = require('path');
 
-export const LATEST_PATH = process.env.VSCODE_LATEST_PATH;
-export const STABLE_PATH = process.env.VSCODE_STABLE_PATH;
-export const WORKSPACE_PATH = process.env.SMOKETEST_REPO;
-export const CODE_WORKSPACE_PATH = process.env.VSCODE_WORKSPACE_PATH;
+export const LATEST_PATH = process.env.VSCODE_PATH || '';
+export const STABLE_PATH = process.env.VSCODE_STABLE_PATH || '';
+export const WORKSPACE_PATH = process.env.SMOKETEST_REPO || '';
+export const CODE_WORKSPACE_PATH = process.env.VSCODE_WORKSPACE_PATH || '';
 export const USER_DIR = 'test_data/temp_user_dir';
 export const EXTENSIONS_DIR = 'test_data/temp_extensions_dir';
 
@@ -50,6 +50,11 @@ export class SpectronApplication {
 			args.push(`--extensions-dir=${this.sampleExtensionsDir}`);
 		}
 
+		const repo = process.env.VSCODE_REPOSITORY;
+		if (repo) {
+			args = [repo, ...args];
+		}
+
 		this.spectron = new Application({
 			path: electronPath,
 			args: args,
@@ -96,7 +101,7 @@ export class SpectronApplication {
 	}
 
 	private retrieveKeybindings() {
-		fs.readFile(path.join(process.cwd(), `test_data/keybindings.json`), 'utf8', (err, data) => {
+		fs.readFile(path.join(__dirname, '../../test_data/keybindings.json'), 'utf8', (err, data) => {
 			if (err) {
 				throw err;
 			}

vscode-smoketest-express @ 5ac7ff24

+Subproject commit 5ac7ff24e8c4028e00d8b7d093a920db31513d2e