testing: update test results api to spec

Refs https://github.com/microsoft/vscode/issues/107467#issuecomment-780776814

src/vs/vscode.proposed.d.ts

@@ -2203,24 +2203,17 @@ declare module 'vscode' {
 		export function createDocumentTestObserver(document: TextDocument): TestObserver;
 
 		/**
-		 * The last or selected test run. Cleared when a new test run starts.
-		 */
-		export const testResults: TestResults | undefined;
+		* List of test results stored by VS Code, sorted in descnding
+		* order by their `completedAt` time.
+		*/
+		export const testResults: ReadonlyArray<TestResults>;
 
 		/**
-		 * Event that fires when the testResults are updated.
-		 */
+		* Event that fires when the {@link testResults} array is updated.
+		*/
 		export const onDidChangeTestResults: Event<void>;
 	}
 
-	export interface TestResults {
-		/**
-		 * The results from the latest test run. The array contains a snapshot of
-		 * all tests involved in the run at the moment when it completed.
-		 */
-		readonly tests: ReadonlyArray<RequiredTestItem> | undefined;
-	}
-
 	export interface TestObserver {
 		/**
 		 * List of tests returned by test provider for files in the workspace.
@@ -2527,6 +2520,45 @@ declare module 'vscode' {
 		location?: Location;
 	}
 
+	/**
+	 * TestResults can be provided to VS Code, or read from it.
+	 *
+	 * The results contain a 'snapshot' of the tests at the point when the test
+	 * run is complete. Therefore, information such as {@link Location} instances
+	 * may be out of date. If the test still exists in the workspace, consumers
+	 * can use its `id` to correlate the result instance with the living test.
+	 *
+	 * @todo coverage and other info may eventually live here
+	 */
+	export interface TestResults {
+		/**
+		 * Unix milliseconds timestamp at which the tests were completed.
+		 */
+		completedAt: number;
+
+		/**
+		 * List of test results. The items in this array are the items that
+		 * were passed in the {@link test.runTests} method.
+		 */
+		results: ReadonlyArray<Readonly<TestItemWithResults>>;
+	}
+
+	/**
+	 * A {@link TestItem} with an associated result, which appear or can be
+	 * provided in {@link TestResult} interfaces.
+	 */
+	export interface TestItemWithResults extends TestItem {
+		/**
+		 * Current result of the test.
+		 */
+		result: TestState;
+
+		/**
+		 * Optional list of nested tests for this item.
+		 */
+		children?: Readonly<TestItemWithResults>[];
+	}
+
 	//#endregion
 
 	//#region Opener service (https://github.com/microsoft/vscode/issues/109277)

src/vs/workbench/api/browser/mainThreadTesting.ts

@@ -5,6 +5,7 @@
 
 import { CancellationToken } from 'vs/base/common/cancellation';
 import { Disposable, IDisposable } from 'vs/base/common/lifecycle';
+import { isDefined } from 'vs/base/common/types';
 import { URI, UriComponents } from 'vs/base/common/uri';
 import { Range } from 'vs/editor/common/core/range';
 import { extHostNamedCustomer } from 'vs/workbench/api/common/extHostCustomers';
@@ -40,11 +41,20 @@ export class MainThreadTesting extends Disposable implements MainThreadTestingSh
 		this._register(this.testService.onShouldSubscribe(args => this.proxy.$subscribeToTests(args.resource, args.uri)));
 		this._register(this.testService.onShouldUnsubscribe(args => this.proxy.$unsubscribeFromTests(args.resource, args.uri)));
 
-		// const testCompleteListener = this._register(new MutableDisposable());
-		// todo(@connor4312): reimplement, maybe
-		// this._register(resultService.onResultsChanged(results => {
-		// testCompleteListener.value = results.onComplete(() => this.proxy.$publishTestResults({ tests: [] }));
-		// }));
+
+		const prevResults = resultService.results.map(r => r.toJSON()).filter(isDefined);
+		if (prevResults.length) {
+			this.proxy.$publishTestResults(prevResults);
+		}
+
+		this._register(resultService.onResultsChanged(evt => {
+			if ('completed' in evt) {
+				const serialized = evt.completed.toJSON();
+				if (serialized) {
+					this.proxy.$publishTestResults([serialized]);
+				}
+			}
+		}));
 
 		testService.updateRootProviderCount(1);
 

src/vs/workbench/api/common/extHost.api.impl.ts

@@ -341,11 +341,11 @@ export function createApiFactoryAndRegisterActors(accessor: ServicesAccessor): I
 			},
 			get onDidChangeTestResults() {
 				checkProposedApiEnabled(extension);
-				return extHostTesting.onLastResultsChanged;
+				return extHostTesting.onResultsChanged;
 			},
 			get testResults() {
 				checkProposedApiEnabled(extension);
-				return extHostTesting.lastResults;
+				return extHostTesting.results;
 			},
 		};
 

src/vs/workbench/api/common/extHost.protocol.ts

@@ -58,7 +58,7 @@ import { ISerializableEnvironmentVariableCollection } from 'vs/workbench/contrib
 import { DebugConfigurationProviderTriggerKind, WorkspaceTrustState } from 'vs/workbench/api/common/extHostTypes';
 import { IAccessibilityInformation } from 'vs/platform/accessibility/common/accessibility';
 import { IExtensionIdWithVersion } from 'vs/platform/userDataSync/common/extensionsStorageSync';
-import { InternalTestItem, InternalTestResults, ITestState, RunTestForProviderRequest, RunTestsRequest, TestIdWithProvider, TestsDiff } from 'vs/workbench/contrib/testing/common/testCollection';
+import { InternalTestItem, ITestState, RunTestForProviderRequest, RunTestsRequest, TestIdWithProvider, TestsDiff, ISerializedTestResults } from 'vs/workbench/contrib/testing/common/testCollection';
 import { CandidatePort } from 'vs/workbench/services/remote/common/remoteExplorerService';
 import { WorkspaceTrustStateChangeEvent } from 'vs/platform/workspace/common/workspaceTrust';
 
@@ -1853,7 +1853,7 @@ export interface ExtHostTestingShape {
 	$unsubscribeFromTests(resource: ExtHostTestingResource, uri: UriComponents): void;
 	$lookupTest(test: TestIdWithProvider): Promise<InternalTestItem | undefined>;
 	$acceptDiff(resource: ExtHostTestingResource, uri: UriComponents, diff: TestsDiff): void;
-	$publishTestResults(results: InternalTestResults): void;
+	$publishTestResults(results: ISerializedTestResults[]): void;
 }
 
 export interface MainThreadTestingShape {

src/vs/workbench/api/common/extHostTesting.ts

@@ -9,6 +9,7 @@ import { CancellationToken } from 'vs/base/common/cancellation';
 import { Emitter } from 'vs/base/common/event';
 import { once } from 'vs/base/common/functional';
 import { DisposableStore, IDisposable } from 'vs/base/common/lifecycle';
+import { deepFreeze } from 'vs/base/common/objects';
 import { isDefined } from 'vs/base/common/types';
 import { URI, UriComponents } from 'vs/base/common/uri';
 import { generateUuid } from 'vs/base/common/uuid';
@@ -20,7 +21,7 @@ import { TestItem, TestState } from 'vs/workbench/api/common/extHostTypeConverte
 import { Disposable } from 'vs/workbench/api/common/extHostTypes';
 import { IExtHostWorkspace } from 'vs/workbench/api/common/extHostWorkspace';
 import { OwnedTestCollection, SingleUseTestCollection } from 'vs/workbench/contrib/testing/common/ownedTestCollection';
-import { AbstractIncrementalTestCollection, IncrementalChangeCollector, IncrementalTestCollectionItem, InternalTestItem, InternalTestItemWithChildren, InternalTestResults, RunTestForProviderRequest, TestDiffOpType, TestIdWithProvider, TestsDiff } from 'vs/workbench/contrib/testing/common/testCollection';
+import { AbstractIncrementalTestCollection, IncrementalChangeCollector, IncrementalTestCollectionItem, InternalTestItem, ISerializedTestResults, RunTestForProviderRequest, SerializedTestResultItem, TestDiffOpType, TestIdWithProvider, TestsDiff } from 'vs/workbench/contrib/testing/common/testCollection';
 import type * as vscode from 'vscode';
 
 const getTestSubscriptionKey = (resource: ExtHostTestingResource, uri: URI) => `${resource}:${uri.toString()}`;
@@ -39,8 +40,8 @@ export class ExtHostTesting implements ExtHostTestingShape {
 	private workspaceObservers: WorkspaceFolderTestObserverFactory;
 	private textDocumentObservers: TextDocumentTestObserverFactory;
 
-	public onLastResultsChanged = this.resultsChangedEmitter.event;
-	public lastResults?: vscode.TestResults;
+	public onResultsChanged = this.resultsChangedEmitter.event;
+	public results: ReadonlyArray<vscode.TestResults> = [];
 
 	constructor(@IExtHostRpcService rpc: IExtHostRpcService, @IExtHostDocumentsAndEditors private readonly documents: IExtHostDocumentsAndEditors, @IExtHostWorkspace private readonly workspace: IExtHostWorkspace) {
 		this.proxy = rpc.getProxy(MainContext.MainThreadTesting);
@@ -105,11 +106,15 @@ export class ExtHostTesting implements ExtHostTestingShape {
 	 * Updates test results shown to extensions.
 	 * @override
 	 */
-	public $publishTestResults(results: InternalTestResults): void {
-		const convert = (item: InternalTestItemWithChildren): vscode.RequiredTestItem =>
-			({ ...TestItem.toShallow(item.item), children: item.children.map(convert) });
+	public $publishTestResults(results: ISerializedTestResults[]): void {
+		this.results = Object.freeze(
+			results
+				.map(r => deepFreeze(convertTestResults(r)))
+				.concat(this.results)
+				.sort((a, b) => b.completedAt - a.completedAt)
+				.slice(0, 32),
+		);
 
-		this.lastResults = { tests: results.tests.map(convert) };
 		this.resultsChangedEmitter.fire();
 	}
 
@@ -350,6 +355,29 @@ export class ExtHostTesting implements ExtHostTestingShape {
 	}
 }
 
+const convertTestResultItem = (item: SerializedTestResultItem, byInternalId: Map<string, SerializedTestResultItem>): vscode.TestItemWithResults => ({
+	...TestItem.toShallow(item.item),
+	result: TestState.to(item.state),
+	children: item.children
+		.map(c => byInternalId.get(c))
+		.filter(isDefined)
+		.map(c => convertTestResultItem(c, byInternalId)),
+});
+
+const convertTestResults = (r: ISerializedTestResults): vscode.TestResults => {
+	const roots: SerializedTestResultItem[] = [];
+	const byInternalId = new Map<string, SerializedTestResultItem>();
+	for (const item of r.items) {
+		byInternalId.set(item.id, item);
+		if (item.direct) {
+			roots.push(item);
+		}
+	}
+
+	return { completedAt: r.completedAt, results: roots.map(r => convertTestResultItem(r, byInternalId)) };
+};
+
+
 /*
  * A class which wraps a vscode.TestItem that provides the ability to filter a TestItem's children
  * to only the children that are located in a certain vscode.Uri.

src/vs/workbench/contrib/testing/browser/testingDecorations.ts

@@ -26,9 +26,9 @@ import { BREAKPOINT_EDITOR_CONTRIBUTION_ID, IBreakpointEditorContribution } from
 import { testingRunAllIcon, testingRunIcon, testingStatesToIcons } from 'vs/workbench/contrib/testing/browser/icons';
 import { TestingOutputPeekController } from 'vs/workbench/contrib/testing/browser/testingOutputPeek';
 import { testMessageSeverityColors } from 'vs/workbench/contrib/testing/browser/theme';
-import { IncrementalTestCollectionItem, IRichLocation, ITestMessage } from 'vs/workbench/contrib/testing/common/testCollection';
+import { IncrementalTestCollectionItem, IRichLocation, ITestMessage, TestResultItem } from 'vs/workbench/contrib/testing/common/testCollection';
 import { buildTestUri, TestUriType } from 'vs/workbench/contrib/testing/common/testingUri';
-import { ITestResultService, TestResultItem } from 'vs/workbench/contrib/testing/common/testResultService';
+import { ITestResultService } from 'vs/workbench/contrib/testing/common/testResultService';
 import { IMainThreadTestCollection, ITestService } from 'vs/workbench/contrib/testing/common/testService';
 
 export class TestingDecorations extends Disposable implements IEditorContribution {

src/vs/workbench/contrib/testing/browser/testingExplorerView.ts

@@ -899,7 +899,7 @@ class TestRunProgress {
 	}
 
 	private updateProgress() {
-		const running = this.resultService.results.filter(r => !r.isComplete);
+		const running = this.resultService.results.filter(r => r.completedAt === undefined);
 		if (!running.length) {
 			this.setIdleText(this.resultService.results[0]?.counts);
 			this.current?.deferred.complete();
@@ -948,7 +948,7 @@ class TestRunProgress {
 			return;
 		}
 
-		if (!result.isComplete) {
+		if (result.completedAt === undefined) {
 			const badge = new ProgressBadge(() => localize('testBadgeRunning', 'Test run in progress'));
 			this.badge.value = this.activityService.showViewActivity(Testing.ExplorerViewId, { badge, clazz: 'progress-badge' });
 			return;

src/vs/workbench/contrib/testing/browser/testingOutputPeek.ts

@@ -31,11 +31,11 @@ import { EditorModel } from 'vs/workbench/common/editor';
 import { testingPeekBorder } from 'vs/workbench/contrib/testing/browser/theme';
 import { AutoOpenPeekViewWhen, getTestingConfiguration, TestingConfigKeys } from 'vs/workbench/contrib/testing/common/configuration';
 import { Testing } from 'vs/workbench/contrib/testing/common/constants';
-import { ITestItem, ITestMessage, ITestState } from 'vs/workbench/contrib/testing/common/testCollection';
+import { ITestItem, ITestMessage, ITestState, TestResultItem } from 'vs/workbench/contrib/testing/common/testCollection';
 import { TestingContextKeys } from 'vs/workbench/contrib/testing/common/testingContextKeys';
 import { isFailedState } from 'vs/workbench/contrib/testing/common/testingStates';
 import { buildTestUri, parseTestUri, TestUriType } from 'vs/workbench/contrib/testing/common/testingUri';
-import { ITestResult, ITestResultService, TestResultItem, TestResultItemChange, TestResultItemChangeReason } from 'vs/workbench/contrib/testing/common/testResultService';
+import { ITestResult, ITestResultService, TestResultItemChange, TestResultItemChangeReason } from 'vs/workbench/contrib/testing/common/testResultService';
 import { IEditorService } from 'vs/workbench/services/editor/common/editorService';
 
 interface ITestDto {

src/vs/workbench/contrib/testing/common/testCollection.ts

@@ -79,12 +79,32 @@ export interface InternalTestItem {
 	item: ITestItem;
 }
 
-export interface InternalTestItemWithChildren extends InternalTestItem {
-	children: this[];
+/**
+ * Test result item used in the main thread.
+ */
+export interface TestResultItem extends IncrementalTestCollectionItem {
+	/** Current state of this test */
+	state: ITestState;
+	/** Computed state based on children */
+	computedState: TestRunState;
+	/** True if the test is outdated */
+	retired: boolean;
+	/** True if the test was directly requested by the run (is not a child or parent) */
+	direct?: true;
 }
 
-export interface InternalTestResults {
-	tests: InternalTestItemWithChildren[];
+export type SerializedTestResultItem = Omit<TestResultItem, 'children' | 'retired'> & { children: string[], retired: undefined };
+
+/**
+ * Test results serialized for transport and storage.
+ */
+export interface ISerializedTestResults {
+	/** ID of these test results */
+	id: string;
+	/** Time the results were compelted */
+	completedAt: number;
+	/** Subset of test result items */
+	items: SerializedTestResultItem[];
 }
 
 export const enum TestDiffOpType {

src/vs/workbench/contrib/testing/common/testResultService.ts

@@ -4,6 +4,8 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { Emitter, Event } from 'vs/base/common/event';
+import { Lazy } from 'vs/base/common/lazy';
+import { isDefined } from 'vs/base/common/types';
 import { URI } from 'vs/base/common/uri';
 import { generateUuid } from 'vs/base/common/uuid';
 import { Range } from 'vs/editor/common/core/range';
@@ -13,7 +15,7 @@ import { IStorageService, StorageScope, StorageTarget } from 'vs/platform/storag
 import { TestRunState } from 'vs/workbench/api/common/extHostTypes';
 import { IComputedStateAccessor, refreshComputedState } from 'vs/workbench/contrib/testing/common/getComputedState';
 import { StoredValue } from 'vs/workbench/contrib/testing/common/storedValue';
-import { IncrementalTestCollectionItem, ITestState, RunTestsRequest } from 'vs/workbench/contrib/testing/common/testCollection';
+import { IncrementalTestCollectionItem, ISerializedTestResults, ITestState, RunTestsRequest, TestResultItem } from 'vs/workbench/contrib/testing/common/testCollection';
 import { TestingContextKeys } from 'vs/workbench/contrib/testing/common/testingContextKeys';
 import { statesInOrder } from 'vs/workbench/contrib/testing/common/testingStates';
 import { IMainThreadTestCollection } from 'vs/workbench/contrib/testing/common/testService';
@@ -47,9 +49,10 @@ export interface ITestResult {
 	readonly id: string;
 
 	/**
-	 * Gets whether the test run has finished.
+	 * If the test is completed, the unix milliseconds time at which it was
+	 * completed. If undefined, the test is still running.
 	 */
-	readonly isComplete: boolean;
+	readonly completedAt: number | undefined;
 
 	/**
 	 * Whether this test result is triggered from an auto run.
@@ -70,7 +73,7 @@ export interface ITestResult {
 	 * Serializes the test result. Used to save and restore results
 	 * in the workspace.
 	 */
-	toJSON(): ISerializedResults;
+	toJSON(): ISerializedTestResults | undefined;
 }
 
 export const makeEmptyCounts = () => {
@@ -174,18 +177,6 @@ const makeNodeAndChildren = (
 	return mapped;
 };
 
-interface ISerializedResults {
-	id: string;
-	items: (Omit<TestResultItem, 'children' | 'retired'> & { children: string[], retired: undefined })[];
-}
-
-export interface TestResultItem extends IncrementalTestCollectionItem {
-	state: ITestState;
-	computedState: TestRunState;
-	retired: boolean;
-	direct?: true;
-}
-
 /**
  * Results of a test. These are created when the test initially started running
  * and marked as "complete" when the run finishes.
@@ -218,7 +209,7 @@ export class LiveTestResult implements ITestResult {
 
 	private readonly completeEmitter = new Emitter<void>();
 	private readonly changeEmitter = new Emitter<TestResultItemChange>();
-	private _complete = false;
+	private _completedAt?: number;
 
 	public readonly onChange = this.changeEmitter.event;
 	public readonly onComplete = this.completeEmitter.event;
@@ -231,8 +222,8 @@ export class LiveTestResult implements ITestResult {
 	/**
 	 * @inheritdoc
 	 */
-	public get isComplete() {
-		return this._complete;
+	public get completedAt() {
+		return this._completedAt;
 	}
 
 	/**
@@ -387,29 +378,32 @@ export class LiveTestResult implements ITestResult {
 	 * Notifies the service that all tests are complete.
 	 */
 	public markComplete() {
-		if (this._complete) {
+		if (this._completedAt !== undefined) {
 			throw new Error('cannot complete a test result multiple times');
 		}
 
 		// un-queue any tests that weren't explicitly updated
 		this.setAllToState(unsetState, t => t.state.state === TestRunState.Queued);
-		this._complete = true;
+		this._completedAt = Date.now();
 		this.completeEmitter.fire();
 	}
 
 	/**
 	 * @inheritdoc
 	 */
-	public toJSON(): ISerializedResults {
-		return {
-			id: this.id,
-			items: [...this.testByExtId.values()].map(entry => ({
-				...entry,
-				retired: undefined,
-				children: [...entry.children],
-			})),
-		};
+	public toJSON(): ISerializedTestResults | undefined {
+		return this.completedAt ? this.doSerialize.getValue() : undefined;
 	}
+
+	private readonly doSerialize = new Lazy((): ISerializedTestResults => ({
+		id: this.id,
+		completedAt: this.completedAt!,
+		items: [...this.testByExtId.values()].map(entry => ({
+			...entry,
+			retired: undefined,
+			children: [...entry.children],
+		})),
+	}));
 }
 
 /**
@@ -429,7 +423,7 @@ class HydratedTestResult implements ITestResult {
 	/**
 	 * @inheritdoc
 	 */
-	public readonly isComplete = true;
+	public readonly completedAt: number;
 
 	/**
 	 * @inheritdoc
@@ -440,8 +434,9 @@ class HydratedTestResult implements ITestResult {
 
 	private readonly byExtId = new Map<string, TestResultItem>();
 
-	constructor(private readonly serialized: ISerializedResults) {
+	constructor(private readonly serialized: ISerializedTestResults) {
 		this.id = serialized.id;
+		this.completedAt = serialized.completedAt;
 
 		for (const item of serialized.items) {
 			const cast: TestResultItem = { ...item, retired: true, children: new Set(item.children) };
@@ -472,7 +467,7 @@ class HydratedTestResult implements ITestResult {
 	/**
 	 * @inheritdoc
 	 */
-	public toJSON(): ISerializedResults {
+	public toJSON(): ISerializedTestResults {
 		return this.serialized;
 	}
 }
@@ -545,7 +540,7 @@ export class TestResultService implements ITestResultService {
 	public readonly onTestChanged = this.testChangeEmitter.event;
 
 	private readonly isRunning: IContextKey<boolean>;
-	private readonly serializedResults: StoredValue<ISerializedResults[]>;
+	private readonly serializedResults: StoredValue<ISerializedTestResults[]>;
 
 	constructor(@IContextKeyService contextKeyService: IContextKeyService, @IStorageService storage: IStorageService) {
 		this.isRunning = TestingContextKeys.isRunning.bindTo(contextKeyService);
@@ -557,7 +552,10 @@ export class TestResultService implements ITestResultService {
 
 		try {
 			for (const value of this.serializedResults.get([])) {
-				this.results.push(new HydratedTestResult(value));
+				// todo@connor4312: temp to migrate old insiders
+				if (value.completedAt) {
+					this.results.push(new HydratedTestResult(value));
+				}
 			}
 		} catch (e) {
 			// outdated structure
@@ -609,7 +607,7 @@ export class TestResultService implements ITestResultService {
 		const keep: ITestResult[] = [];
 		const removed: ITestResult[] = [];
 		for (const result of this.results) {
-			if (result.isComplete) {
+			if (result.completedAt !== undefined) {
 				removed.push(result);
 			} else {
 				keep.push(result);
@@ -617,20 +615,19 @@ export class TestResultService implements ITestResultService {
 		}
 
 		this.results = keep;
-		this.serializedResults.store(this.results.map(r => r.toJSON()));
+		this.serializedResults.store(this.results.map(r => r.toJSON()).filter(isDefined));
 		this.changeResultEmitter.fire({ removed });
 	}
 
 	private onComplete(result: LiveTestResult) {
 		// move the complete test run down behind any still-running ones
-		for (let i = 0; i < this.results.length - 1; i++) {
-			if (this.results[i].isComplete && !this.results[i + 1].isComplete) {
-				[this.results[i], this.results[i + 1]] = [this.results[i + 1], this.results[i]];
-			}
-		}
-
-		this.isRunning.set(!this.results[0]?.isComplete);
-		this.serializedResults.store(this.results.map(r => r.toJSON()));
+		this.resort();
+		this.isRunning.set(this.results.length > 0 && this.results[0].completedAt === undefined);
+		this.serializedResults.store(this.results.map(r => r.toJSON()).filter(isDefined));
 		this.changeResultEmitter.fire({ completed: result });
 	}
+
+	private resort() {
+		this.results.sort((a, b) => (b.completedAt ?? Number.MAX_SAFE_INTEGER) - (a.completedAt ?? Number.MAX_SAFE_INTEGER));
+	}
 }

src/vs/workbench/contrib/testing/test/common/testResultService.test.ts

@@ -161,7 +161,7 @@ suite('Workbench - Test Results Service', () => {
 
 			assert.deepStrictEqual(actual, { ...expected, retired: true });
 			assert.deepStrictEqual(rehydrated.counts, r.counts);
-			assert.strictEqual(rehydrated.isComplete, true);
+			assert.strictEqual(typeof rehydrated.completedAt, 'number');
 		});
 
 		test('clears results but keeps ongoing tests', () => {