From 547cf4e81f1262042915dcf7494ab03b5030b15e Mon Sep 17 00:00:00 2001 From: Daniel Imms Date: Mon, 3 Feb 2020 02:26:55 -0800 Subject: [PATCH] Run smoke tests against actual build (#83799) * Run web against actual server Part of #80308 * Fix strict null check errors * Fix folder arg * Disable unit tests and integration tests temporarily * Allow running on node 12 * Fix smoke test condition * Disable continue on error * Add web to server dir * fix smoke test to use build for web * enable in product build Co-authored-by: Benjamin Pasero --- .../darwin/product-build-darwin.yml | 20 ++--- test/automation/src/puppeteerDriver.ts | 10 ++- test/smoke/src/main.ts | 88 +++++++++++-------- 3 files changed, 70 insertions(+), 48 deletions(-) diff --git a/build/azure-pipelines/darwin/product-build-darwin.yml b/build/azure-pipelines/darwin/product-build-darwin.yml index 273a728927f..50c85822a53 100644 --- a/build/azure-pipelines/darwin/product-build-darwin.yml +++ b/build/azure-pipelines/darwin/product-build-darwin.yml @@ -114,16 +114,16 @@ steps: displayName: Run integration tests condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false')) -# Web Smoke Tests disabled due to https://github.com/microsoft/vscode/issues/80308 -# - script: | -# set -e -# cd test/smoke -# yarn compile -# cd - -# yarn smoketest --web --headless -# continueOnError: true -# displayName: Run web smoke tests -# condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false')) +- script: | + set -e + cd test/smoke + yarn compile + cd - + VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-web-darwin" \ + yarn smoketest --web --headless + continueOnError: true + displayName: Run web smoke tests + condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false')) - script: | set -e diff --git a/test/automation/src/puppeteerDriver.ts b/test/automation/src/puppeteerDriver.ts index 40fd699b6d3..73a118b6043 100644 --- a/test/automation/src/puppeteerDriver.ts +++ b/test/automation/src/puppeteerDriver.ts @@ -98,8 +98,14 @@ export async function launch(_args: string[]): Promise { VSCODE_AGENT_FOLDER: agentFolder, ...process.env }; + let serverLocation: string | undefined; + if (process.env.VSCODE_REMOTE_SERVER_PATH) { + serverLocation = join(process.env.VSCODE_REMOTE_SERVER_PATH, `server.${process.platform === 'win32' ? 'cmd' : 'sh'}`); + } else { + serverLocation = join(args[0], `resources/server/web.${process.platform === 'win32' ? 'bat' : 'sh'}`); + } server = spawn( - join(args[0], `resources/server/web.${process.platform === 'win32' ? 'bat' : 'sh'}`), + serverLocation, ['--browser', 'none', '--driver', 'web'], { env } ); @@ -140,7 +146,7 @@ export function connect(headless: boolean, outPath: string, handle: string): Pro }); const page = (await browser.pages())[0]; await page.setViewport({ width, height }); - await page.goto(`${endpoint}&folder=${args![1]}`); + await page.goto(`${endpoint}&folder=vscode-remote://localhost:9888${args![1]}`); const result = { client: { dispose: () => teardown }, driver: buildDriver(browser, page) diff --git a/test/smoke/src/main.ts b/test/smoke/src/main.ts index 0e34ec32cd9..1d84795e1b9 100644 --- a/test/smoke/src/main.ts +++ b/test/smoke/src/main.ts @@ -37,8 +37,8 @@ import { setup as setupDataMultirootTests } from './areas/multiroot/multiroot.te import { setup as setupDataLocalizationTests } from './areas/workbench/localization.test'; import { setup as setupLaunchTests } from './areas/workbench/launch.test'; -if (!/^v10/.test(process.version)) { - console.error('Error: Smoketest must be run using Node 10. Currently running', process.version); +if (!/^v10/.test(process.version) && !/^v12/.test(process.version)) { + console.error('Error: Smoketest must be run using Node 10/12. Currently running', process.version); process.exit(1); } @@ -73,7 +73,6 @@ const extensionsPath = path.join(testDataPath, 'extensions-dir'); mkdirp.sync(extensionsPath); const screenshotsPath = opts.screenshots ? path.resolve(opts.screenshots) : null; - if (screenshotsPath) { mkdirp.sync(screenshotsPath); } @@ -83,10 +82,6 @@ function fail(errorMessage): void { process.exit(1); } -if (parseInt(process.version.substr(1)) < 6) { - fail('Please update your Node version to greater than 6 to run the smoke test.'); -} - const repoPath = path.join(__dirname, '..', '..', '..'); function getDevElectronPath(): string { @@ -122,44 +117,65 @@ function getBuildElectronPath(root: string): string { } } -let testCodePath = opts.build; -let stableCodePath = opts['stable-build']; -let electronPath: string; -let stablePath: string | undefined = undefined; +let quality: Quality; + +if (!opts.web) { + let testCodePath = opts.build; + let stableCodePath = opts['stable-build']; + let electronPath: string; + let stablePath: string | undefined = undefined; + + if (testCodePath) { + electronPath = getBuildElectronPath(testCodePath); + + if (stableCodePath) { + stablePath = getBuildElectronPath(stableCodePath); + } + } else { + testCodePath = getDevElectronPath(); + electronPath = testCodePath; + process.env.VSCODE_REPOSITORY = repoPath; + process.env.VSCODE_DEV = '1'; + process.env.VSCODE_CLI = '1'; + } + + if (!fs.existsSync(electronPath || '')) { + fail(`Can't find VSCode at ${electronPath}.`); + } -if (testCodePath) { - electronPath = getBuildElectronPath(testCodePath); + if (typeof stablePath === 'string' && !fs.existsSync(stablePath)) { + fail(`Can't find Stable VSCode at ${stablePath}.`); + } - if (stableCodePath) { - stablePath = getBuildElectronPath(stableCodePath); + if (process.env.VSCODE_DEV === '1') { + quality = Quality.Dev; + } else if (electronPath.indexOf('Code - Insiders') >= 0 /* macOS/Windows */ || electronPath.indexOf('code-insiders') /* Linux */ >= 0) { + quality = Quality.Insiders; + } else { + quality = Quality.Stable; } } else { - testCodePath = getDevElectronPath(); - electronPath = testCodePath; - process.env.VSCODE_REPOSITORY = repoPath; - process.env.VSCODE_DEV = '1'; - process.env.VSCODE_CLI = '1'; -} + let testCodeServerPath = process.env.VSCODE_REMOTE_SERVER_PATH; -if (!opts.web && !fs.existsSync(electronPath || '')) { - fail(`Can't find Code at ${electronPath}.`); -} + if (typeof testCodeServerPath === 'string' && !fs.existsSync(testCodeServerPath)) { + fail(`Can't find Code server at ${testCodeServerPath}.`); + } + + if (!testCodeServerPath) { + process.env.VSCODE_REPOSITORY = repoPath; + process.env.VSCODE_DEV = '1'; + process.env.VSCODE_CLI = '1'; + } -if (typeof stablePath === 'string' && !fs.existsSync(stablePath)) { - fail(`Can't find Stable Code at ${stablePath}.`); + if (process.env.VSCODE_DEV === '1') { + quality = Quality.Dev; + } else { + quality = Quality.Insiders; + } } const userDataDir = path.join(testDataPath, 'd'); -let quality: Quality; -if (process.env.VSCODE_DEV === '1') { - quality = Quality.Dev; -} else if (electronPath.indexOf('Code - Insiders') >= 0 /* macOS/Windows */ || electronPath.indexOf('code-insiders') /* Linux */ >= 0) { - quality = Quality.Insiders; -} else { - quality = Quality.Stable; -} - async function setupRepository(): Promise { if (opts['test-repo']) { console.log('*** Copying test project repository:', opts['test-repo']); @@ -246,7 +262,7 @@ after(async function () { }); if (!opts.web) { - setupDataMigrationTests(stableCodePath, testDataPath); + setupDataMigrationTests(opts['stable-build'], testDataPath); } describe('Running Code', () => { -- GitLab