diff --git a/build/npm/postinstall.js b/build/npm/postinstall.js index 7b156212bec8208be15c649eadcb985b7cf723f0..63e675e773321413d0bdb20c5eca810b1b7e534c 100644 --- a/build/npm/postinstall.js +++ b/build/npm/postinstall.js @@ -58,4 +58,5 @@ function npmInstallBuildDependencies() { } npmInstall(`build`); // node modules required for build +npmInstall('test/smoke'); // node modules required for smoketest npmInstallBuildDependencies(); // node modules for watching, specific to host node version, not electron \ No newline at end of file diff --git a/build/tfs/darwin/smoketest.sh b/build/tfs/darwin/smoketest.sh index ad1606c3aad870c5df975b169b8c535488d6d9e3..e666fdec69ec94270f3fea2ab6523cf39cc61415 100755 --- a/build/tfs/darwin/smoketest.sh +++ b/build/tfs/darwin/smoketest.sh @@ -24,5 +24,5 @@ step "Build minified & upload source maps" \ step "Run smoke test" \ pushd test/smoke npm install - npm test -- --latest "$AGENT_BUILDDIRECTORY/VSCode-darwin/Visual Studio Code - Insiders.app/Contents/MacOS/Electron" + npm run smoketest -- "$AGENT_BUILDDIRECTORY/VSCode-darwin/Visual Studio Code - Insiders.app/Contents/MacOS/Electron" popd \ No newline at end of file diff --git a/build/tfs/linux/smoketest.sh b/build/tfs/linux/smoketest.sh index 8d71fff127508764369baef93aede3064f49eb34..e5d34e9e7f43eb9c39d4b0adacea25194f2beab9 100644 --- a/build/tfs/linux/smoketest.sh +++ b/build/tfs/linux/smoketest.sh @@ -36,7 +36,7 @@ function configureEnvironment { function runTest { pushd test/smoke npm install - sudo -u testuser -H xvfb-run -a -s "-screen 0 1024x768x8" npm test -- --latest "$AGENT_BUILDDIRECTORY/VSCode-linux-ia32/code-insiders" + sudo -u testuser -H xvfb-run -a -s "-screen 0 1024x768x8" npm run smoketest -- "$AGENT_BUILDDIRECTORY/VSCode-linux-ia32/code-insiders" popd } diff --git a/build/tfs/win32/smoketest.ps1 b/build/tfs/win32/smoketest.ps1 index 2a6519aaaf4f4a2173a1346754d3e6b91bd16035..f5d25867103143098a89c8b0ba80141eca0396b9 100644 --- a/build/tfs/win32/smoketest.ps1 +++ b/build/tfs/win32/smoketest.ps1 @@ -40,7 +40,7 @@ step "Build minified" { step "Run smoke test" { exec { & Push-Location test\smoke } exec { & npm install } - exec { & npm test -- --latest "$env:AGENT_BUILDDIRECTORY\VSCode-win32-$global:arch\Code - Insiders.exe" } + exec { & npm run smoketest -- "$env:AGENT_BUILDDIRECTORY\VSCode-win32-$global:arch\Code - Insiders.exe" } exec { & Pop-Location } } diff --git a/package.json b/package.json index f0a3ac6e339f4fb30af89dc2cf38f9be7141acbb..3628d7afc2a6acd0c0731c24e877bb5aadd99c14 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,8 @@ "precommit": "node build/gulpfile.hygiene.js", "gulp": "gulp --max_old_space_size=4096", "7z": "7z", - "update-grammars": "node build/npm/update-all-grammars.js" + "update-grammars": "node build/npm/update-all-grammars.js", + "smoketest": "node test/smoke/out/main.js" }, "dependencies": { "applicationinsights": "0.17.1", diff --git a/test/smoke/README.md b/test/smoke/README.md index 02717c281fbc5341937e5066d783c97e6c5e11fc..a22a039e95e42368abceda2409dea94eae9dac0c 100644 --- a/test/smoke/README.md +++ b/test/smoke/README.md @@ -1,9 +1,14 @@ # VS Code Smoke Testing -- Run `npm install` -- Start the tests: `npm test -- --latest "path/to/binary"`. +``` +npm rum smoketest -- "path/to/code" +``` -If you want to include 'Data Migration' area tests use `npm test -- --latest path/to/binary --stable path/to/currentStable` respectively. +If you want to include 'Data Migration' area tests use: + +``` +npm run smoketest -- "path/to/code" "path/to/codeStable" +``` Detailed prerequisites and running steps are described [in our smoke test wiki](https://github.com/Microsoft/vscode/wiki/Smoke-Test#automated-smoke-test). @@ -50,15 +55,7 @@ To add new test, `./test/${area}.ts` should be updated. The same instruction-sty Almost on every automated test action it captures a screenshot. These help to determine an issue, if smoke test fails. The normal workflow is that you understand what code is doing and then try to match it up with screenshots obtained from the test. # Running "Out of Sources" -If you did a fix in VS Code that you need in order for the smoke test to succeed, here is how you can run the smoke test against the sources of VS Code: -* Set related environment variables in the console: - * `export NODE_ENV=development` - * `export VSCODE_DEV=1` - * `export VSCODE_CLI=1` -* open `application.ts` - * pass in the vscode folder as argument to the application - * e.g. instead of `args: args` type `args: ['/Users/bpasero/Development/vscode', ...args]` -* `cd test/smoke` -* `npm install` -* `npm test -- --latest ` - * e.g. on macOS: `npm test -- --latest /.build/electron/Code\ -\ OSS.app/Contents/MacOS/Electron` \ No newline at end of file + +``` +npm run smoketest +``` \ No newline at end of file diff --git a/test/smoke/package.json b/test/smoke/package.json index b5d425c29ae600044abf9c5344d0815ec4aaa418..d8bc57d6bc14a353b75296f8e7e607bb0d4f4b8a 100644 --- a/test/smoke/package.json +++ b/test/smoke/package.json @@ -3,24 +3,22 @@ "version": "0.1.0", "main": "./src/main.js", "scripts": { - "compile": "tsc", - "pretest": "tsc", - "test": "node out/main.js" + "postinstall": "tsc", + "watch": "tsc --watch" }, "devDependencies": { - "@types/mocha": "^2.2.41", - "@types/node": "^6.0.70", - "@types/webdriverio": "^4.6.1", "@types/electron": "~1.4.37", - "@types/rimraf": "^0.0.28", "@types/htmlparser2": "^3.7.29", + "@types/mkdirp": "^0.5.1", + "@types/mocha": "^2.2.41", + "@types/node": "^8.0.26", + "@types/rimraf": "^0.0.28", + "@types/webdriverio": "^4.6.1", + "htmlparser2": "^3.9.2", "mocha": "^3.2.0", - "spectron": "~3.6.4", - "typescript": "^2.2.2", "rimraf": "^2.6.1", - "commander": "^2.9.0", - "simple-git": "^1.73.0", + "spectron": "~3.6.4", "strip-json-comments": "^2.0.1", - "htmlparser2": "^3.9.2" + "typescript": "^2.2.2" } } \ No newline at end of file diff --git a/test/smoke/src/main.ts b/test/smoke/src/main.ts index 0be92d92fb0fac66867ce00bcee530bb93802f98..2f0c4f4790031eeeb6bbe4a0ba24a7f313115357 100644 --- a/test/smoke/src/main.ts +++ b/test/smoke/src/main.ts @@ -3,220 +3,132 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -const fs = require('fs'); -const https = require('https'); -const program = require('commander'); -const git = require('simple-git')(); -const child_process = require('child_process'); -const path = require('path'); -const mkdirp = require('mkdirp'); - -const testDataPath = path.join(process.cwd(), 'test_data'); -const codeWorkspacePath = path.join(testDataPath, 'smoketest.code-workspace'); +import * as fs from 'fs'; +import * as https from 'https'; +import * as cp from 'child_process'; +import * as path from 'path'; +import * as mkdirp from 'mkdirp'; + +const testDataPath = path.join(__dirname, '..', 'test_data'); +const workspacePath = path.join(testDataPath, 'smoketest.code-workspace'); const testRepoUrl = 'https://github.com/Microsoft/vscode-smoketest-express'; const testRepoLocalDir = path.join(testDataPath, 'vscode-smoketest-express'); -const keybindingsUrl = 'https://raw.githubusercontent.com/Microsoft/vscode-docs/master/scripts/keybindings'; - mkdirp.sync(testDataPath); -program - .option('-l, --latest ', 'path to the latest VS Code to test') - .option('-s, --stable [file path]', 'path to the stable VS Code to be used in data migration tests'); - -program.on('--help', () => { - console.log(' Examples:'); - console.log(''); - console.log(' $ npm test -- --latest path/to/binary'); - console.log(' $ npm test -- -l path/to/binary'); - console.log(''); - console.log(' $ npm test -- --latest path/to/latest/binary --stable path/to/stable/binary'); - console.log(' $ npm test -- -l path/to/latest/binary -s path/to/stable/binary'); - console.log(''); -}); -program.parse(process.argv); - -if (!program.latest) { - fail('You must specify the binary to run the smoke test against'); -} -if (!binaryExists(program.latest) || (program.stable && !binaryExists(program.stable))) { - fail('The file path to electron binary does not exist or permissions do not allow to execute it. Please check the path provided.'); +function fail(errorMessage): void { + console.error(errorMessage); + process.exit(1); } + if (parseInt(process.version.substr(1)) < 6) { fail('Please update your Node version to greater than 6 to run the smoke test.'); } -// Setting up environment variables -process.env.VSCODE_LATEST_PATH = program.latest; -if (program.stable) { - process.env.VSCODE_STABLE_PATH = program.stable; -} -process.env.SMOKETEST_REPO = testRepoLocalDir; -if (program.latest && (program.latest.indexOf('Code - Insiders') /* macOS/Windows */ || program.latest.indexOf('code-insiders') /* Linux */) >= 0) { - process.env.VSCODE_EDITION = 'insiders'; +const repoPath = path.join(__dirname, '..', '..', '..'); + +function getDevElectronPath(): string { + const buildPath = path.join(repoPath, '.build'); + const product = require(path.join(repoPath, 'product.json')); + + switch (process.platform) { + case 'darwin': + return path.join(buildPath, `${product.nameLong}.app`, 'Contents', 'MacOS', 'Electron'); + case 'linux': + return path.join(buildPath, 'electron', `${product.applicationName}`); + case 'win32': + return path.join(buildPath, 'electron', `${product.nameShort}.exe`); + default: + throw new Error('Unsupported platform.'); + } } -process.env.VSCODE_WORKSPACE_PATH = codeWorkspacePath; -// Setting up 'vscode-smoketest-express' project -let os = process.platform.toString(); -if (os === 'darwin') { - os = 'osx'; +let [, , testCodePath, stableCodePath] = process.argv; + +if (testCodePath) { + process.env.VSCODE_PATH = testCodePath; + + if (stableCodePath) { + process.env.VSCODE_STABLE_PATH = stableCodePath; + } +} else { + testCodePath = getDevElectronPath(); + process.env.VSCODE_PATH = testCodePath; + process.env.VSCODE_REPOSITORY = repoPath; + process.env.VSCODE_DEV = '1'; + process.env.VSCODE_CLI = '1'; } -else if (os === 'win32') { - os = 'win'; + +if (!fs.existsSync(testCodePath)) { + fail(`Can't find Code at ${testCodePath}.`); } -main().catch(err => console.error(err)); +process.env.SMOKETEST_REPO = testRepoLocalDir; +process.env.VSCODE_WORKSPACE_PATH = workspacePath; -async function main(): Promise { - await getKeybindings(`${keybindingsUrl}/doc.keybindings.${os}.json`, path.join(testDataPath, 'keybindings.json')); - - const workspace = { - id: (Date.now() + Math.round(Math.random() * 1000)).toString(), - folders: [ - toUri(path.join(testRepoLocalDir, 'public')), - toUri(path.join(testRepoLocalDir, 'routes')), - toUri(path.join(testRepoLocalDir, 'views')) - ] - }; - - await createWorkspaceFile(codeWorkspacePath, workspace); - await cleanOrClone(testRepoUrl, testRepoLocalDir); - await execute('npm install', testRepoLocalDir); - await runTests(); +if ((testCodePath.indexOf('Code - Insiders') /* macOS/Windows */ || testCodePath.indexOf('code-insiders') /* Linux */) >= 0) { + process.env.VSCODE_EDITION = 'insiders'; } -function fail(errorMessage): void { - console.error(errorMessage); - process.exit(1); +function getKeybindingPlatform(): string { + switch (process.platform) { + case 'darwin': return 'osx'; + case 'win32': return 'win'; + default: return process.platform; + } } function toUri(path: string): string { - if (os === 'win') { + if (process.platform === 'win32') { return `file:///${path.replace(/\\/g, '/')}`; } return `file://${path}`; } -function runTests(): void { - console.log('Running tests...'); - var proc = child_process.spawn(process.execPath, [ - 'out/mocha-runner.js' - ]); - proc.stdout.on('data', data => { - console.log(data.toString()); - }); - proc.stderr.on('data', data => { - var date = new Date().toLocaleString(); - fs.appendFile(path.join(testDataPath, 'errors.log'), `${date}: ${data.toString()}`, (err) => { - if (err) { - throw new Error(`Could not write stderr to errors.log with the following error: ${err}`); - }; - }); - }); - proc.on('exit', (code) => { - process.exit(code); +async function main(): Promise { + const keybindingsUrl = `https://raw.githubusercontent.com/Microsoft/vscode-docs/master/scripts/keybindings/doc.keybindings.${getKeybindingPlatform()}.json`; + console.log(`Fetching keybindings from ${keybindingsUrl}...`); + + await new Promise((c, e) => { + https.get(keybindingsUrl, res => { + const output = fs.createWriteStream(path.join(testDataPath, 'keybindings.json')); + res.on('error', e); + output.on('error', e); + output.on('close', c); + res.pipe(output); + }).on('error', e); }); -} -async function cleanOrClone(repo: string, dir: string): Promise { - console.log('Cleaning or cloning test project repository...'); + if (!fs.existsSync(workspacePath)) { + console.log('Creating workspace file...'); + const workspace = { + id: (Date.now() + Math.round(Math.random() * 1000)).toString(), + folders: [ + toUri(path.join(testRepoLocalDir, 'public')), + toUri(path.join(testRepoLocalDir, 'routes')), + toUri(path.join(testRepoLocalDir, 'views')) + ] + }; + + fs.writeFileSync(workspacePath, JSON.stringify(workspace, null, '\t')); + } - if (!folderExists(dir)) { - await gitClone(repo, dir); + if (!fs.existsSync(testRepoLocalDir)) { + console.log('Cloning test project repository...'); + cp.spawnSync('git', ['clone', testRepoUrl, testRepoLocalDir]); } else { - git.cwd(dir); - await new Promise((c, e) => git.fetch(err => err ? e(err) : c())); - await gitResetAndClean(); + console.log('Cleaning test project repository...'); + cp.spawnSync('git', ['fetch'], { cwd: testRepoLocalDir }); + cp.spawnSync('git', ['reset', '--hard', 'FETCH_HEAD'], { cwd: testRepoLocalDir }); + cp.spawnSync('git', ['clean', '-xdf'], { cwd: testRepoLocalDir }); } -} - -function gitClone(repo: string, dir: string): Promise { - return new Promise((res, rej) => { - git.clone(repo, dir, () => { - console.log('Test repository successfully cloned.'); - res(); - }); - }); -} -async function gitResetAndClean(): Promise { - await new Promise((c, e) => git.reset(['FETCH_HEAD', '--hard'], err => err ? e(err) : c())); - await new Promise((c, e) => git.clean('f', ['-d'], err => err ? e(err) : c())); - console.log('Test project was successfully reset to initial state.'); -} + console.log('Running npm install...'); + cp.execSync('npm install', { cwd: testRepoLocalDir, stdio: 'inherit' }); -function execute(cmd: string, dir: string): Promise { - return new Promise((res, rej) => { - console.log(`Running ${cmd}...`); - child_process.exec(cmd, { cwd: dir, stdio: [0, 1, 2] }, (error, stdout, stderr) => { - if (error) { - rej(error); - } - if (stderr) { - console.error(stderr); - } - console.log(stdout); - res(); - }); - }); -} - -function getKeybindings(url: string, location: string): Promise { - console.log(`Fetching keybindings from ${url}...`); - return new Promise((resolve, reject) => { - https.get(url, (res) => { - if (res.statusCode !== 200) { - reject(`Failed to obtain key bindings with response code: ${res.statusCode}`); - } - - var buffer: Buffer[] = []; - res.on('data', (chunk) => buffer.push(chunk)); - res.on('end', () => { - fs.writeFile(location, Buffer.concat(buffer), 'utf8', () => { - console.log('Keybindings were successfully fetched.'); - resolve(); - }); - }); - }).on('error', (e) => { - reject(`Failed to obtain key bindings with an error: ${e}`); - }); - }); -} - -function createWorkspaceFile(path: string, workspace: any): Promise { - console.log(`Creating workspace file at ${path}...`); - return new Promise((resolve, reject) => { - fs.exists(path, exists => { - if (exists) { - return resolve(); - } - - fs.writeFile(path, JSON.stringify(workspace, null, '\t'), error => { - if (error) { - reject(error); - } else { - resolve(); - } - }); - }); - }); -} - -function folderExists(folder: string): boolean { - try { - fs.accessSync(folder, 'rw'); - return true; - } catch (e) { - return false; - } + console.log('Running tests...'); + const mocha = cp.spawnSync(process.execPath, ['out/mocha-runner.js'], { cwd: path.join(__dirname, '..'), stdio: 'inherit' }); + process.exit(mocha.status); } -function binaryExists(filePath: string): boolean { - try { - fs.accessSync(filePath, 'x'); - return true; - } catch (e) { - return false; - } -} \ No newline at end of file +main().catch(fail); \ No newline at end of file diff --git a/test/smoke/src/mocha-runner.ts b/test/smoke/src/mocha-runner.ts index a711a84ca0e825ce5aa9b120752064d91ddf73e3..fb7f0bb79fb03ebbbefbf1f04bf09f36184b7bb3 100644 --- a/test/smoke/src/mocha-runner.ts +++ b/test/smoke/src/mocha-runner.ts @@ -4,13 +4,14 @@ *--------------------------------------------------------------------------------------------*/ const MochaTest = require('mocha'); +const path = require('path'); const mochaTest = new MochaTest({ timeout: 60000, slow: 10000, useColors: true }); -mochaTest.addFile(require('path').join(process.cwd(), 'out/test.js')); +mochaTest.addFile(path.join(__dirname, 'test.js')); mochaTest.run((failures) => { process.exit(failures); }); \ No newline at end of file diff --git a/test/smoke/src/spectron/application.ts b/test/smoke/src/spectron/application.ts index 76766356bc523cfc4e4f7bede6ffae0776f122b6..dad6d171269012ac7f2c0c9b20da7dc9c6cf34a6 100644 --- a/test/smoke/src/spectron/application.ts +++ b/test/smoke/src/spectron/application.ts @@ -9,10 +9,10 @@ import { Screenshot } from '../helpers/screenshot'; var fs = require('fs'); var path = require('path'); -export const LATEST_PATH = process.env.VSCODE_LATEST_PATH; -export const STABLE_PATH = process.env.VSCODE_STABLE_PATH; -export const WORKSPACE_PATH = process.env.SMOKETEST_REPO; -export const CODE_WORKSPACE_PATH = process.env.VSCODE_WORKSPACE_PATH; +export const LATEST_PATH = process.env.VSCODE_PATH || ''; +export const STABLE_PATH = process.env.VSCODE_STABLE_PATH || ''; +export const WORKSPACE_PATH = process.env.SMOKETEST_REPO || ''; +export const CODE_WORKSPACE_PATH = process.env.VSCODE_WORKSPACE_PATH || ''; export const USER_DIR = 'test_data/temp_user_dir'; export const EXTENSIONS_DIR = 'test_data/temp_extensions_dir'; @@ -50,6 +50,11 @@ export class SpectronApplication { args.push(`--extensions-dir=${this.sampleExtensionsDir}`); } + const repo = process.env.VSCODE_REPOSITORY; + if (repo) { + args = [repo, ...args]; + } + this.spectron = new Application({ path: electronPath, args: args, @@ -96,7 +101,7 @@ export class SpectronApplication { } private retrieveKeybindings() { - fs.readFile(path.join(process.cwd(), `test_data/keybindings.json`), 'utf8', (err, data) => { + fs.readFile(path.join(__dirname, '../../test_data/keybindings.json'), 'utf8', (err, data) => { if (err) { throw err; } diff --git a/test_data/vscode-smoketest-express b/test_data/vscode-smoketest-express new file mode 160000 index 0000000000000000000000000000000000000000..5ac7ff24e8c4028e00d8b7d093a920db31513d2e --- /dev/null +++ b/test_data/vscode-smoketest-express @@ -0,0 +1 @@ +Subproject commit 5ac7ff24e8c4028e00d8b7d093a920db31513d2e