smoke: simplify screenshots, artifacts

build/tfs/darwin/build.sh

@@ -38,12 +38,10 @@ step "Run integration tests" \
 	./scripts/test-integration.sh
 
 function smoketest {
-	SCREENSHOTS="$AGENT_BUILDDIRECTORY/smoketest-screenshots"
-	LOGS="$AGENT_BUILDDIRECTORY/smoketest-logs"
-	rm -rf $SCREENSHOTS $LOGS
-	mkdir -p $SCREENSHOTS $LOGS
+	ARTIFACTS="$AGENT_BUILDDIRECTORY/smoketest-artifacts"
+	rm -rf $ARTIFACTS
 
-	npm run smoketest -- --build "$AGENT_BUILDDIRECTORY/VSCode-darwin/Visual Studio Code - Insiders.app"  --screenshots $SCREENSHOTS --logs $LOGS
+	npm run smoketest -- --build "$AGENT_BUILDDIRECTORY/VSCode-darwin/Visual Studio Code - Insiders.app" --debug $ARTIFACTS
 }
 
 step "Run smoke test" \

build/tfs/linux/build.sh

@@ -40,21 +40,16 @@ step "Run unit tests" \
 	./scripts/test.sh --build --reporter dot
 
 function smoketest {
- 	SCREENSHOTS="$AGENT_BUILDDIRECTORY/smoketest-screenshots"
-	rm -rf $SCREENSHOTS
-	mkdir -p $SCREENSHOTS
-
- 	LOGS="$AGENT_BUILDDIRECTORY/smoketest-logs"
-	rm -rf $LOGS
-	mkdir -p $LOGS
-
 	id -u testuser &>/dev/null || (useradd -m testuser; chpasswd <<< testuser:testpassword)
 	sudo -i -u testuser -- sh -c 'git config --global user.name "VS Code Agent" &&  git config --global user.email "monacotools@microsoft.com"'
-	chown -R testuser $SCREENSHOTS
-	chown -R testuser $LOGS
+
+ 	ARTIFACTS="$AGENT_BUILDDIRECTORY/smoketest-artifacts"
+	rm -rf $ARTIFACTS
+	mkdir -p $ARTIFACTS
+	chown -R testuser $ARTIFACTS
 
 	ps -o pid= -u testuser | xargs sudo kill -9
-	DISPLAY=:10 sudo -i -u testuser -- sh -c "cd $BUILD_SOURCESDIRECTORY/test/smoke && ./node_modules/.bin/mocha --build $AGENT_BUILDDIRECTORY/VSCode-linux-$ARCH --screenshots $SCREENSHOTS --logs $LOGS"
+	DISPLAY=:10 sudo -i -u testuser -- sh -c "cd $BUILD_SOURCESDIRECTORY/test/smoke && ./node_modules/.bin/mocha --build $AGENT_BUILDDIRECTORY/VSCode-linux-$ARCH --debug $ARTIFACTS"
 	# DISPLAY=:10 sudo -i -u testuser -- sh -c "cd /vso/work/1/s/test/smoke && ./node_modules/.bin/mocha --build /vso/work/1/VSCode-linux-ia32"
 }
 

build/tfs/win32/1_build.ps1

@@ -53,13 +53,10 @@ step "Run unit tests" {
 # }
 
 step "Run smoke test" {
-	$Screenshots = "$env:AGENT_BUILDDIRECTORY\smoketest-screenshots"
-	Remove-Item -Recurse -Force -ErrorAction Ignore $Screenshots
+	$Artifacts = "$env:AGENT_BUILDDIRECTORY\smoketest-artifacts"
+	Remove-Item -Recurse -Force -ErrorAction Ignore $Artifacts
 
-	$Logs = "$env:AGENT_BUILDDIRECTORY\smoketest-logs"
-	Remove-Item -Recurse -Force -ErrorAction Ignore $Logs
-
-	exec { & npm run smoketest -- --build "$env:AGENT_BUILDDIRECTORY\VSCode-win32-$global:arch" --screenshots "$Screenshots" --logs "$Logs" }
+	exec { & npm run smoketest -- --build "$env:AGENT_BUILDDIRECTORY\VSCode-win32-$global:arch" --debug "$Artifacts" }
 }
 
 done

build/tfs/win32/build_unsigned.ps1

@@ -48,13 +48,10 @@ step "Run unit tests" {
 }
 
 step "Run smoke test" {
-	$Screenshots = "$env:AGENT_BUILDDIRECTORY\smoketest-screenshots"
-	Remove-Item -Recurse -Force -ErrorAction Ignore $Screenshots
+	$Artifacts = "$env:AGENT_BUILDDIRECTORY\smoketest-artifacts"
+	Remove-Item -Recurse -Force -ErrorAction Ignore $Artifacts
 
-	$Logs = "$env:AGENT_BUILDDIRECTORY\smoketest-logs"
-	Remove-Item -Recurse -Force -ErrorAction Ignore $Logs
-
-	exec { & npm run smoketest -- --build "$env:AGENT_BUILDDIRECTORY\VSCode-win32-$global:arch" --screenshots "$Screenshots" --logs "$Logs" }
+	exec { & npm run smoketest -- --build "$env:AGENT_BUILDDIRECTORY\VSCode-win32-$global:arch" --debug "$Artifacts" }
 }
 
 step "Create archive and setup package" {

test/smoke/src/areas/css/css.test.ts

@@ -11,7 +11,6 @@ describe('CSS', () => {
 	let app: SpectronApplication;
 	before(function () { app = new SpectronApplication(); return app.start('CSS'); });
 	after(() => app.stop());
-	beforeEach(function () { app.screenCapturer.testName = this.currentTest.title; });
 
 	it('verifies quick outline', async () => {
 		await app.workbench.quickopen.openFile('style.css');

test/smoke/src/areas/debug/debug.test.ts

@@ -42,7 +42,6 @@ describe('Debug', () => {
 	// otherwise concurrent test runs will clash on those ports
 	before(async () => await app.start('Debug', [], { PORT: String(await findFreePort()), ...process.env }));
 	after(() => app.stop());
-	beforeEach(function () { app.screenCapturer.testName = this.currentTest.title; });
 
 	it('configure launch json', async function () {
 		await app.workbench.debug.openDebugViewlet();

test/smoke/src/areas/editor/editor.test.ts

@@ -11,7 +11,6 @@ describe('Editor', () => {
 	let app: SpectronApplication;
 	before(() => { app = new SpectronApplication(); return app.start('Editor'); });
 	after(() => app.stop());
-	beforeEach(function () { app.screenCapturer.testName = this.currentTest.title; });
 
 	it('shows correct quick outline', async function () {
 		await app.workbench.quickopen.openFile('www');

test/smoke/src/areas/explorer/explorer.test.ts

@@ -9,7 +9,6 @@ describe('Explorer', () => {
 	let app: SpectronApplication;
 	before(() => { app = new SpectronApplication(); return app.start('Explorer'); });
 	after(() => app.stop());
-	beforeEach(function () { app.screenCapturer.testName = this.currentTest.title; });
 
 	it('quick open search produces correct result', async function () {
 		const expectedNames = [

test/smoke/src/areas/extensions/extensions.test.ts

@@ -10,7 +10,6 @@ describe('Extensions', () => {
 	let app: SpectronApplication = new SpectronApplication();
 	before(() => app.start('Extensions'));
 	after(() => app.stop());
-	beforeEach(function () { app.screenCapturer.testName = this.currentTest.title; });
 
 	if (app.build !== VSCODE_BUILD.DEV) {
 		it(`install and activate vscode-smoketest-check extension`, async function () {

test/smoke/src/areas/git/git.test.ts

@@ -14,7 +14,6 @@ describe('Git', () => {
 	let app: SpectronApplication;
 	before(() => { app = new SpectronApplication(); return app.start('Git'); });
 	after(() => app.stop());
-	beforeEach(function () { app.screenCapturer.testName = this.currentTest.title; });
 
 	it('reflects working tree changes', async function () {
 		await app.workbench.scm.openSCMViewlet();

test/smoke/src/areas/multiroot/multiroot.test.ts

@@ -5,7 +5,6 @@
 
 import * as assert from 'assert';
 import { SpectronApplication, CODE_WORKSPACE_PATH, VSCODE_BUILD } from '../../spectron/application';
-import { Window } from '../window';
 
 describe('Multiroot', () => {
 	let app: SpectronApplication = new SpectronApplication(void 0, CODE_WORKSPACE_PATH);
@@ -15,7 +14,6 @@ describe('Multiroot', () => {
 
 	before(() => app.start('Multi Root'));
 	after(() => app.stop());
-	beforeEach(function () { app.screenCapturer.testName = this.currentTest.title; });
 
 	it('shows results from all folders', async function () {
 		await app.workbench.quickopen.openQuickOpen();
@@ -26,7 +24,7 @@ describe('Multiroot', () => {
 	});
 
 	it('shows workspace name in title', async function () {
-		const title = await new Window(app).getTitle();
+		const title = await app.client.getTitle();
 		await app.screenCapturer.capture('window title');
 		assert.ok(title.indexOf('smoketest (Workspace)') >= 0);
 	});

test/smoke/src/areas/preferences/preferences.test.ts

@@ -12,7 +12,6 @@ describe('Preferences', () => {
 	let app: SpectronApplication;
 	before(() => { app = new SpectronApplication(); return app.start('Preferences'); });
 	after(() => app.stop());
-	beforeEach(function () { app.screenCapturer.testName = this.currentTest.title; });
 
 	it('turns off editor line numbers and verifies the live change', async function () {
 		await app.workbench.explorer.openFile('app.js');

test/smoke/src/areas/search/search.test.ts

@@ -9,7 +9,6 @@ describe('Search', () => {
 	let app: SpectronApplication;
 	before(() => { app = new SpectronApplication(); return app.start('Search'); });
 	after(() => app.stop());
-	beforeEach(function () { app.screenCapturer.testName = this.currentTest.title; });
 
 	it('searches for body & checks for correct result number', async function () {
 		await app.workbench.search.openSearchViewlet();

test/smoke/src/areas/statusbar/statusbar.test.ts

@@ -13,7 +13,6 @@ describe('Statusbar', () => {
 	let app: SpectronApplication = new SpectronApplication();
 	before(() => app.start('Statusbar'));
 	after(() => app.stop());
-	beforeEach(function () { app.screenCapturer.testName = this.currentTest.title; });
 
 	it('verifies presence of all default status bar elements', async function () {
 		await app.workbench.statusbar.waitForStatusbarElement(StatusBarElement.BRANCH_STATUS);

test/smoke/src/areas/terminal/terminal.test.ts

@@ -9,7 +9,6 @@ describe('Terminal', () => {
 	let app: SpectronApplication;
 	before(() => { app = new SpectronApplication(); return app.start('Terminal'); });
 	after(() => app.stop());
-	beforeEach(function () { app.screenCapturer.testName = this.currentTest.title; });
 
 	it(`opens terminal, runs 'echo' and verifies the output`, async function () {
 		const expected = new Date().getTime().toString();

test/smoke/src/areas/window.ts

-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the MIT License. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-
-import { SpectronApplication } from '../spectron/application';
-
-export class Window {
-
-	constructor(private spectron: SpectronApplication) {
-
-	}
-
-	public async getTitle(): Promise<string> {
-		return this.spectron.client.getTitle();
-	}
-
-}

test/smoke/src/areas/workbench/data-loss.test.ts

@@ -9,7 +9,6 @@ describe('Dataloss', () => {
 	let app: SpectronApplication;
 	before(() => { app = new SpectronApplication(); return app.start('Dataloss'); });
 	after(() => app.stop());
-	beforeEach(function () { app.screenCapturer.testName = this.currentTest.title; });
 
 	it(`verifies that 'hot exit' works for dirty files`, async function () {
 		await app.workbench.newUntitledFile();

test/smoke/src/areas/workbench/data-migration.test.ts

@@ -23,7 +23,6 @@ describe('Data Migration', () => {
 		// Setting up stable version
 		let app = new SpectronApplication(STABLE_PATH);
 		await app.start('Data Migration');
-		app.screenCapturer.testName = 'Untitled is restorted';
 
 		await app.workbench.newUntitledFile();
 		await app.workbench.editor.waitForTypeInEditor('Untitled-1', textToType);
@@ -34,7 +33,6 @@ describe('Data Migration', () => {
 
 		app = new SpectronApplication(LATEST_PATH);
 		await app.start('Data Migration');
-		app.screenCapturer.testName = 'Untitled is restorted';
 
 		assert.ok(await app.workbench.waitForActiveTab('Untitled-1', true), `Untitled-1 tab is not present after migration.`);
 
@@ -50,7 +48,6 @@ describe('Data Migration', () => {
 		let app = new SpectronApplication(STABLE_PATH, fileName);
 		await Util.removeFile(`${fileName}`);
 		await app.start('Data Migration');
-		app.screenCapturer.testName = 'Newly created dirty file is restorted';
 
 		await app.workbench.waitForActiveTab(fileName);
 		await app.client.type(firstTextPart);
@@ -63,7 +60,6 @@ describe('Data Migration', () => {
 		// Checking latest version for the restored state
 		app = new SpectronApplication(LATEST_PATH);
 		await app.start('Data Migration');
-		app.screenCapturer.testName = 'Newly created dirty file is restorted';
 
 		const filename = fileName.split('/')[1];
 		assert.ok(await app.workbench.waitForActiveTab(filename), `Untitled-1 tab is not present after migration.`);
@@ -76,7 +72,6 @@ describe('Data Migration', () => {
 		const fileName1 = 'app.js', fileName2 = 'jsconfig.json', fileName3 = 'readme.md';
 		let app = new SpectronApplication(STABLE_PATH);
 		await app.start('Data Migration');
-		app.screenCapturer.testName = 'Opened tabs are restored';
 
 		await app.workbench.quickopen.openFile(fileName1);
 		await app.workbench.quickopen.openFile(fileName2);
@@ -85,7 +80,6 @@ describe('Data Migration', () => {
 
 		app = new SpectronApplication(LATEST_PATH);
 		await app.start('Data Migration');
-		app.screenCapturer.testName = 'Opened tabs are restored';
 
 		assert.ok(await app.workbench.waitForTab(fileName1), `${fileName1} tab was not restored after migration.`);
 		assert.ok(await app.workbench.waitForTab(fileName2), `${fileName2} tab was not restored after migration.`);

test/smoke/src/areas/workbench/localization.test.ts

@@ -17,7 +17,6 @@ describe('Localization', () => {
 
 	it(`starts with 'DE' locale and verifies title and viewlets text is in German`, async function () {
 		await app.start('Localization', ['--locale=DE']);
-		app.screenCapturer.testName = 'DE locale test';
 
 		let text = await app.workbench.explorer.getOpenEditorsViewTitle();
 		await app.screenCapturer.capture('Open editors title');

test/smoke/src/helpers/screenshot.ts

@@ -5,35 +5,29 @@
 
 import * as path from 'path';
 import * as fs from 'fs';
-import * as mkdirp from 'mkdirp';
 import { Application } from 'spectron';
-import { SCREENSHOTS_DIR } from '../spectron/application';
-
-function sanitize(name: string): string {
-	return name.replace(/[&*:\/]/g, '');
-}
+import { sanitize } from './utilities';
 
 export class ScreenCapturer {
 
 	private static counter = 0;
-	testName: string = 'default';
 
-	constructor(private application: Application, private suiteName: string) { }
+	constructor(
+		private application: Application,
+		private screenshotsDirPath: string | undefined
+	) { }
 
 	async capture(name: string): Promise<void> {
-		if (!SCREENSHOTS_DIR) {
+		if (!this.screenshotsDirPath) {
 			return;
 		}
 
 		const screenshotPath = path.join(
-			SCREENSHOTS_DIR,
-			sanitize(this.suiteName),
-			sanitize(this.testName),
+			this.screenshotsDirPath,
 			`${ScreenCapturer.counter++}-${sanitize(name)}.png`
 		);
 
 		const image = await this.application.browserWindow.capturePage();
-		await new Promise((c, e) => mkdirp(path.dirname(screenshotPath), err => err ? e(err) : c()));
 		await new Promise((c, e) => fs.writeFile(screenshotPath, image, err => err ? e(err) : c()));
 	}
 }

test/smoke/src/helpers/utilities.ts

@@ -75,4 +75,8 @@ export async function mkdirp(path: string, mode?: number): Promise<boolean> {
 	}
 
 	return true;
+}
+
+export function sanitize(name: string): string {
+	return name.replace(/[&*:\/]/g, '');
 }
\ No newline at end of file

test/smoke/src/main.ts

@@ -21,14 +21,12 @@ const opts = minimist(args, {
 	string: [
 		'build',
 		'stable-build',
-		'screenshots',
-		'logs',
+		'debug',
 		'wait-time'
 	]
 });
 
-opts.screenshots = opts.screenshots === '' ? path.join(testDataPath, 'screenshots') : opts.screenshots;
-opts.logs = opts.logs === '' ? path.join(testDataPath, 'logs') : opts.logs;
+process.env.ARTIFACTS_DIR = opts.debug || '';
 
 const workspacePath = path.join(testDataPath, 'smoketest.code-workspace');
 const testRepoUrl = 'https://github.com/Microsoft/vscode-smoketest-express';
@@ -107,8 +105,6 @@ process.env.VSCODE_EXTENSIONS_DIR = extensionsPath;
 process.env.SMOKETEST_REPO = testRepoLocalDir;
 process.env.VSCODE_WORKSPACE_PATH = workspacePath;
 process.env.VSCODE_KEYBINDINGS_PATH = keybindingsPath;
-process.env.SCREENSHOTS_DIR = opts.screenshots || '';
-process.env.LOGS_DIR = opts.logs || '';
 process.env.WAIT_TIME = opts['wait-time'] || '20';
 
 if (process.env.VSCODE_DEV === '1') {

test/smoke/src/spectron/application.ts

@@ -12,6 +12,7 @@ import * as fs from 'fs';
 import * as cp from 'child_process';
 import * as path from 'path';
 import * as mkdirp from 'mkdirp';
+import { sanitize } from '../helpers/utilities';
 
 export const LATEST_PATH = process.env.VSCODE_PATH as string;
 export const STABLE_PATH = process.env.VSCODE_STABLE_PATH || '';
@@ -20,8 +21,7 @@ export const CODE_WORKSPACE_PATH = process.env.VSCODE_WORKSPACE_PATH as string;
 export const USER_DIR = process.env.VSCODE_USER_DIR as string;
 export const EXTENSIONS_DIR = process.env.VSCODE_EXTENSIONS_DIR as string;
 export const VSCODE_EDITION = process.env.VSCODE_EDITION as string;
-export const SCREENSHOTS_DIR = process.env.SCREENSHOTS_DIR as string;
-export const LOGS_DIR = process.env.LOGS_DIR as string;
+export const ARTIFACTS_DIR = process.env.ARTIFACTS_DIR as string;
 export const WAIT_TIME = parseInt(process.env.WAIT_TIME as string);
 
 export enum VSCODE_BUILD {
@@ -158,19 +158,30 @@ export class SpectronApplication {
 			requireName: 'nodeRequire'
 		};
 
-		if (LOGS_DIR) {
-			const dir = path.join(LOGS_DIR, id);
-			opts.chromeDriverLogPath = path.join(dir, 'chromedriver.log');
-			mkdirp.sync(dir);
+		let screenshotsDirPath: string | undefined = undefined;
 
-			opts.webdriverLogPath = path.join(dir, 'webdriver');
-			mkdirp.sync(opts.webdriverLogPath);
+		if (ARTIFACTS_DIR) {
+			const testsuiteRootPath = path.join(ARTIFACTS_DIR, sanitize(testSuiteName));
+			mkdirp.sync(testsuiteRootPath);
+
+			// Collect screenshots
+			screenshotsDirPath = path.join(testsuiteRootPath, 'screenshots');
+			mkdirp.sync(screenshotsDirPath);
+
+			// Collect chromedriver logs
+			const chromedriverLogPath = path.join(testsuiteRootPath, 'chromedriver.log');
+			opts.chromeDriverLogPath = chromedriverLogPath;
+
+			// Collect webdriver logs
+			const webdriverLogsPath = path.join(testsuiteRootPath, 'webdriver');
+			mkdirp.sync(webdriverLogsPath);
+			opts.webdriverLogPath = webdriverLogsPath;
 		}
 
 		this.spectron = new Application(opts);
 		await this.spectron.start();
 
-		this._screenCapturer = new ScreenCapturer(this.spectron, testSuiteName);
+		this._screenCapturer = new ScreenCapturer(this.spectron, screenshotsDirPath);
 		this._client = new SpectronClient(this.spectron, this);
 		this._workbench = new Workbench(this);
 	}