Run smoke tests against actual build (#83799)

* Run web against actual server

Part of #80308

* Fix strict null check errors

* Fix folder arg

* Disable unit tests and integration tests temporarily

* Allow running on node 12

* Fix smoke test condition

* Disable continue on error

* Add web to server dir

* fix smoke test to use build for web

* enable in product build
Co-authored-by: NBenjamin Pasero <benjamin.pasero@gmail.com>

build/azure-pipelines/darwin/product-build-darwin.yml

@@ -114,16 +114,16 @@ steps:
   displayName: Run integration tests
   condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
 
-# Web Smoke Tests disabled due to https://github.com/microsoft/vscode/issues/80308
-# - script: |
-#     set -e
-#     cd test/smoke
-#     yarn compile
-#     cd -
-#     yarn smoketest --web --headless
-#   continueOnError: true
-#   displayName: Run web smoke tests
-#   condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
+- script: |
+    set -e
+    cd test/smoke
+    yarn compile
+    cd -
+    VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-web-darwin" \
+    yarn smoketest --web --headless
+  continueOnError: true
+  displayName: Run web smoke tests
+  condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
 
 - script: |
     set -e

test/automation/src/puppeteerDriver.ts

@@ -98,8 +98,14 @@ export async function launch(_args: string[]): Promise<void> {
 		VSCODE_AGENT_FOLDER: agentFolder,
 		...process.env
 	};
+	let serverLocation: string | undefined;
+	if (process.env.VSCODE_REMOTE_SERVER_PATH) {
+		serverLocation = join(process.env.VSCODE_REMOTE_SERVER_PATH, `server.${process.platform === 'win32' ? 'cmd' : 'sh'}`);
+	} else {
+		serverLocation = join(args[0], `resources/server/web.${process.platform === 'win32' ? 'bat' : 'sh'}`);
+	}
 	server = spawn(
-		join(args[0], `resources/server/web.${process.platform === 'win32' ? 'bat' : 'sh'}`),
+		serverLocation,
 		['--browser', 'none', '--driver', 'web'],
 		{ env }
 	);
@@ -140,7 +146,7 @@ export function connect(headless: boolean, outPath: string, handle: string): Pro
 		});
 		const page = (await browser.pages())[0];
 		await page.setViewport({ width, height });
-		await page.goto(`${endpoint}&folder=${args![1]}`);
+		await page.goto(`${endpoint}&folder=vscode-remote://localhost:9888${args![1]}`);
 		const result = {
 			client: { dispose: () => teardown },
 			driver: buildDriver(browser, page)

test/smoke/src/main.ts

@@ -37,8 +37,8 @@ import { setup as setupDataMultirootTests } from './areas/multiroot/multiroot.te
 import { setup as setupDataLocalizationTests } from './areas/workbench/localization.test';
 import { setup as setupLaunchTests } from './areas/workbench/launch.test';
 
-if (!/^v10/.test(process.version)) {
-	console.error('Error: Smoketest must be run using Node 10. Currently running', process.version);
+if (!/^v10/.test(process.version) && !/^v12/.test(process.version)) {
+	console.error('Error: Smoketest must be run using Node 10/12. Currently running', process.version);
 	process.exit(1);
 }
 
@@ -73,7 +73,6 @@ const extensionsPath = path.join(testDataPath, 'extensions-dir');
 mkdirp.sync(extensionsPath);
 
 const screenshotsPath = opts.screenshots ? path.resolve(opts.screenshots) : null;
-
 if (screenshotsPath) {
 	mkdirp.sync(screenshotsPath);
 }
@@ -83,10 +82,6 @@ function fail(errorMessage): void {
 	process.exit(1);
 }
 
-if (parseInt(process.version.substr(1)) < 6) {
-	fail('Please update your Node version to greater than 6 to run the smoke test.');
-}
-
 const repoPath = path.join(__dirname, '..', '..', '..');
 
 function getDevElectronPath(): string {
@@ -122,44 +117,65 @@ function getBuildElectronPath(root: string): string {
 	}
 }
 
-let testCodePath = opts.build;
-let stableCodePath = opts['stable-build'];
-let electronPath: string;
-let stablePath: string | undefined = undefined;
+let quality: Quality;
+
+if (!opts.web) {
+	let testCodePath = opts.build;
+	let stableCodePath = opts['stable-build'];
+	let electronPath: string;
+	let stablePath: string | undefined = undefined;
+
+	if (testCodePath) {
+		electronPath = getBuildElectronPath(testCodePath);
+
+		if (stableCodePath) {
+			stablePath = getBuildElectronPath(stableCodePath);
+		}
+	} else {
+		testCodePath = getDevElectronPath();
+		electronPath = testCodePath;
+		process.env.VSCODE_REPOSITORY = repoPath;
+		process.env.VSCODE_DEV = '1';
+		process.env.VSCODE_CLI = '1';
+	}
+
+	if (!fs.existsSync(electronPath || '')) {
+		fail(`Can't find VSCode at ${electronPath}.`);
+	}
 
-if (testCodePath) {
-	electronPath = getBuildElectronPath(testCodePath);
+	if (typeof stablePath === 'string' && !fs.existsSync(stablePath)) {
+		fail(`Can't find Stable VSCode at ${stablePath}.`);
+	}
 
-	if (stableCodePath) {
-		stablePath = getBuildElectronPath(stableCodePath);
+	if (process.env.VSCODE_DEV === '1') {
+		quality = Quality.Dev;
+	} else if (electronPath.indexOf('Code - Insiders') >= 0 /* macOS/Windows */ || electronPath.indexOf('code-insiders') /* Linux */ >= 0) {
+		quality = Quality.Insiders;
+	} else {
+		quality = Quality.Stable;
 	}
 } else {
-	testCodePath = getDevElectronPath();
-	electronPath = testCodePath;
-	process.env.VSCODE_REPOSITORY = repoPath;
-	process.env.VSCODE_DEV = '1';
-	process.env.VSCODE_CLI = '1';
-}
+	let testCodeServerPath = process.env.VSCODE_REMOTE_SERVER_PATH;
 
-if (!opts.web && !fs.existsSync(electronPath || '')) {
-	fail(`Can't find Code at ${electronPath}.`);
-}
+	if (typeof testCodeServerPath === 'string' && !fs.existsSync(testCodeServerPath)) {
+		fail(`Can't find Code server at ${testCodeServerPath}.`);
+	}
+
+	if (!testCodeServerPath) {
+		process.env.VSCODE_REPOSITORY = repoPath;
+		process.env.VSCODE_DEV = '1';
+		process.env.VSCODE_CLI = '1';
+	}
 
-if (typeof stablePath === 'string' && !fs.existsSync(stablePath)) {
-	fail(`Can't find Stable Code at ${stablePath}.`);
+	if (process.env.VSCODE_DEV === '1') {
+		quality = Quality.Dev;
+	} else {
+		quality = Quality.Insiders;
+	}
 }
 
 const userDataDir = path.join(testDataPath, 'd');
 
-let quality: Quality;
-if (process.env.VSCODE_DEV === '1') {
-	quality = Quality.Dev;
-} else if (electronPath.indexOf('Code - Insiders') >= 0 /* macOS/Windows */ || electronPath.indexOf('code-insiders') /* Linux */ >= 0) {
-	quality = Quality.Insiders;
-} else {
-	quality = Quality.Stable;
-}
-
 async function setupRepository(): Promise<void> {
 	if (opts['test-repo']) {
 		console.log('*** Copying test project repository:', opts['test-repo']);
@@ -246,7 +262,7 @@ after(async function () {
 });
 
 if (!opts.web) {
-	setupDataMigrationTests(stableCodePath, testDataPath);
+	setupDataMigrationTests(opts['stable-build'], testDataPath);
 }
 
 describe('Running Code', () => {