Merge pull request #548 from bpasero/cli-tests

Allow to run extension tests from command line and hook up to build

.travis.yml

@@ -26,7 +26,9 @@ before_install:
   - npm config set python `which python`
   - npm install -g gulp
   - if [ $TRAVIS_OS_NAME == "linux" ]; then
-      export CXX="g++-4.9" CC="gcc-4.9";
+      export CXX="g++-4.9" CC="gcc-4.9" DISPLAY=:99.0;
+      sh -e /etc/init.d/xvfb start;
+      sleep 3;
     fi
 
 install:

extensions/vscode-api-tests/.vscodeignore

 .vscode/**
 typings/**
-out/test/**
-test/**
 **/*.ts
 **/*.map
 .gitignore
 tsconfig.json
-vsc-extension-quickstart.md

extensions/vscode-api-tests/README.md

-# README
-## This is the README for your extension "vscode-api-tests" 
--------------------
-You can author your README using Visual Studio Code.  Here are some useful editor keyboard shortcuts:
-
-* Split the editor (`Cmd+\` on OSX or `Ctrl+\` on Windows and Linux)
-* Toggle preview (`Shift+CMD+V` on OSX or `Shift+Ctrl+V` on Windows and Linux)
-* Press `Ctrl+Space` (Windows, Linux) or `Cmd+Space` (OSX) to see a list of Markdown snippets
-
-### For more information
-* [Visual Studio Code's Markdown Support](http://code.visualstudio.com/docs/languages/markdown)
-* [Markdown Syntax Reference](https://help.github.com/articles/markdown-basics/)
-
-** Enjoy!**

extensions/vscode-api-tests/package.json

 {
 	"name": "vscode-api-tests",
-	"description": "",
+	"description": "API tests for VS Code",
 	"version": "0.0.1",
 	"publisher": "vscode",
+	"private": true,
 	"engines": {
 		"vscode": "*"
 	},

package.json

@@ -12,7 +12,7 @@
 	},
 	"scripts": {
 		"test": "node node_modules/mocha/bin/_mocha",
-		"postinstall": "npm --prefix extensions/csharp-o/ install extensions/csharp-o/"
+		"postinstall": "npm --prefix extensions/csharp-o/ install extensions/csharp-o/ && npm --prefix extensions/vscode-api-tests/ install extensions/vscode-api-tests/"
 	},
 	"dependencies": {
 		"applicationinsights": "0.15.6",

src/vs/workbench/electron-main/env.ts

@@ -106,6 +106,9 @@ if (!fs.existsSync(userPluginsHome)) {
 	fs.mkdirSync(userPluginsHome);
 }
 
+// Helper to identify if we have plugin tests to run from the command line without debugger
+export const isTestingFromCli = cliArgs.pluginTestsPath && !cliArgs.debugBrkPluginHost;
+
 export function log(...a: any[]): void {
 	if (cliArgs.verboseLogging) {
 		console.log.apply(null, a);

src/vs/workbench/electron-main/lifecycle.ts

@@ -67,6 +67,7 @@ export class Lifecycle {
 
 			// Windows/Linux: we quit when all windows have closed
 			// Mac: we only quit when quit was requested
+			// Tests: we always quit
 			if (this.quitRequested || process.platform !== 'darwin') {
 				app.quit();
 			}

src/vs/workbench/electron-main/windows.ts

@@ -85,6 +85,11 @@ export interface IOpenedPathsList {
 	files: string[];
 }
 
+interface ILogEntry {
+	severity: string;
+	arguments: any;
+}
+
 export class WindowsManager {
 
 	public static autoSaveDelayStorageKey = 'autoSaveDelay';
@@ -121,7 +126,7 @@ export class WindowsManager {
 	}
 
 	private registerListeners(): void {
-		app.on('activate', (event:Event, hasVisibleWindows:boolean) => {
+		app.on('activate', (event: Event, hasVisibleWindows: boolean) => {
 			env.log('App#activate');
 
 			// Mac only event: reopen last window when we get activated
@@ -247,7 +252,23 @@ export class WindowsManager {
 			if (broadcast.channel && broadcast.payload) {
 				this.sendToAll('vscode:broadcast', broadcast, [windowId]);
 			}
-		})
+		});
+
+		ipc.on('vscode:log', (event: Event, logEntry: ILogEntry) => {
+			let args = [];
+			try {
+				let parsed = JSON.parse(logEntry.arguments);
+				args.push(...Object.getOwnPropertyNames(parsed).map(o => parsed[o]));
+			} catch (error) {
+				args.push(logEntry.arguments);
+			}
+
+			console[logEntry.severity].apply(console, args);
+		});
+
+		ipc.on('vscode:exit', (event: Event, code: number) => {
+			process.exit(code);
+		});
 
 		UpdateManager.on('update-downloaded', (update: IUpdate) => {
 			this.sendToFocused('vscode:telemetry', { eventName: 'update:downloaded', data: { version: update.version } });

src/vs/workbench/node/pluginHostMain.ts

@@ -99,7 +99,7 @@ export function createServices(remoteCom: IPluginsIPC, initData: IInitData, shar
 }
 
 interface ITestRunner {
-	run(testsRoot:string, clb: (error:Error) => void): void;
+	run(testsRoot:string, clb: (error:Error, failures?: number) => void): void;
 }
 
 export class PluginHostMain {
@@ -235,7 +235,7 @@ export class PluginHostMain {
 		// Execute the runner if it follows our spec
 		if (testRunner && typeof testRunner.run === 'function') {
 			return new TPromise<void>((c, e) => {
-				testRunner.run(env.pluginTestsPath, (error) => {
+				testRunner.run(env.pluginTestsPath, (error, failures) => {
 					if (error) {
 						e(error.toString());
 					} else {
@@ -243,22 +243,22 @@ export class PluginHostMain {
 					}
 
 					// after tests have run, we shutdown the host
-					this.gracefulExit();
+					this.gracefulExit(failures && failures > 0 ? 1 /* ERROR */ : 0 /* OK */);
 				});
 			});
 		}
 
 		// Otherwise make sure to shutdown anyway even in case of an error
 		else {
-			this.gracefulExit();
+			this.gracefulExit(1 /* ERROR */);
 		}
 
 		return TPromise.wrapError<void>(requireError ? requireError.toString() : nls.localize('pluginTestError', "Path {0} does not point to a valid extension test runner.", env.pluginTestsPath));
 	}
 
-	private gracefulExit(): void {
+	private gracefulExit(code: number): void {
 		// to give the PH process a chance to flush any outstanding console
 		// messages to the main process, we delay the exit() by some time
-		setTimeout(() => exit(), 500);
+		setTimeout(() => exit(code), 500);
 	}
 }
\ No newline at end of file

src/vs/workbench/services/thread/electron-browser/threadService.ts

@@ -107,6 +107,7 @@ class PluginHostProcessManager {
 	public startPluginHostProcess(onPluginHostMessage: (msg: any) => void): void {
 		let config = this.contextService.getConfiguration();
 		let isDev = !config.env.isBuilt || !!config.env.pluginDevelopmentPath;
+		let isTestingFromCli = !!config.env.pluginTestsPath && !config.env.debugBrkPluginHost;
 
 		let opts: any = {
 			env: objects.mixin(objects.clone(process.env), { AMD_ENTRYPOINT: 'vs/workbench/node/pluginHostProcess', PIPE_LOGGING: 'true', VERBOSE_LOGGING: true })
@@ -184,11 +185,18 @@ class PluginHostProcessManager {
 							consoleArgs = ['%c[Plugin Host]', 'color: blue', ...args];
 						}
 
-						// Send to local console
-						console[logEntry.severity].apply(console, consoleArgs);
+						// Send to local console unless we run tests from cli
+						if (!isTestingFromCli) {
+							console[logEntry.severity].apply(console, consoleArgs);
+						}
+
+						// Log on main side if running tests from cli
+						if (isTestingFromCli) {
+							ipc.send('vscode:log', logEntry);
+						}
 
 						// Broadcast to other windows if we are in development mode
-						if (isDev) {
+						else if (isDev) {
 							this.windowService.broadcast({
 								channel: PLUGIN_LOG_BROADCAST_CHANNEL,
 								payload: logEntry
@@ -229,9 +237,14 @@ class PluginHostProcessManager {
 						}
 
 						// Expected development plugin termination: When the plugin host goes down we also shutdown the window
-						else {
+						else if (!isTestingFromCli) {
 							this.windowService.getWindow().close();
 						}
+
+						// When CLI testing make sure to exit with proper exit code
+						else {
+							ipc.send('vscode:exit', code);
+						}
 					}
 				});
 			});

test/run.sh

@@ -7,6 +7,7 @@ else
 	ROOT=$(dirname $(dirname $(readlink -f $0)))
 fi
 
+# Unit Tests
 if [[ "$OSTYPE" == "darwin"* ]]; then
 	cd $ROOT ; ulimit -n 4096 ; ATOM_SHELL_INTERNAL_RUN_AS_NODE=1 \
 		../Electron-Build/Electron.app/Contents/MacOS/Electron \
@@ -16,3 +17,6 @@ else
 		../Electron-Build/electron \
 		node_modules/mocha/bin/_mocha $*
 fi
+
+# Integration Tests (currently not enabled for linux because of missing display)
+./scripts/code.sh $ROOT/extensions/vscode-api-tests/testWorkspace --extensionDevelopmentPath=$ROOT/extensions/vscode-api-tests --extensionTestsPath=$ROOT/extensions/vscode-api-tests/out
\ No newline at end of file