Add test for case to assert that fetch should be scheduled again when a text...

Add test for case to assert that fetch should be scheduled again when a text buffer change occurs while the provider runs and the provider returns null

src/vs/editor/common/services/modelServiceImpl.ts

@@ -712,7 +712,9 @@ class SemanticTokensResponse {
 	}
 }
 
-class ModelSemanticColoring extends Disposable {
+export class ModelSemanticColoring extends Disposable {
+
+	public static FETCH_DOCUMENT_SEMANTIC_TOKENS_DELAY = 300;
 
 	private _isDisposed: boolean;
 	private readonly _model: ITextModel;
@@ -728,7 +730,7 @@ class ModelSemanticColoring extends Disposable {
 		this._isDisposed = false;
 		this._model = model;
 		this._semanticStyling = stylingProvider;
-		this._fetchDocumentSemanticTokens = this._register(new RunOnceScheduler(() => this._fetchDocumentSemanticTokensNow(), 300));
+		this._fetchDocumentSemanticTokens = this._register(new RunOnceScheduler(() => this._fetchDocumentSemanticTokensNow(), ModelSemanticColoring.FETCH_DOCUMENT_SEMANTIC_TOKENS_DELAY));
 		this._currentDocumentResponse = null;
 		this._currentDocumentRequestCancellationTokenSource = null;
 		this._documentProvidersChangeListeners = [];
@@ -844,9 +846,7 @@ class ModelSemanticColoring extends Disposable {
 		const rescheduleIfNeeded = () => {
 			if (pendingChanges.length > 0 && !this._fetchDocumentSemanticTokens.isScheduled()) {
 				this._fetchDocumentSemanticTokens.schedule();
-				return true;
 			}
-			return false;
 		};
 
 		if (this._currentDocumentResponse) {
@@ -865,10 +865,8 @@ class ModelSemanticColoring extends Disposable {
 			return;
 		}
 		if (!tokens) {
-			// Only reset semantic tokens if we did not reschedule to avoid "blinking" tokens
-			if (!rescheduleIfNeeded()) {
-				this._model.setSemanticTokens(null, true);
-			}
+			this._model.setSemanticTokens(null, true);
+			rescheduleIfNeeded();
 			return;
 		}
 
@@ -945,8 +943,7 @@ class ModelSemanticColoring extends Disposable {
 			}
 
 			this._model.setSemanticTokens(result, true);
-		}
-		else {
+		} else {
 			this._model.setSemanticTokens(null, true);
 		}
 

src/vs/editor/test/common/services/modelService.test.ts

@@ -11,18 +11,27 @@ import { EditOperation } from 'vs/editor/common/core/editOperation';
 import { Range } from 'vs/editor/common/core/range';
 import { Selection } from 'vs/editor/common/core/selection';
 import { createStringBuilder } from 'vs/editor/common/core/stringBuilder';
-import { DefaultEndOfLine } from 'vs/editor/common/model';
+import { DefaultEndOfLine, ITextModel } from 'vs/editor/common/model';
 import { createTextBuffer } from 'vs/editor/common/model/textModel';
-import { ModelServiceImpl } from 'vs/editor/common/services/modelServiceImpl';
+import { ModelSemanticColoring, ModelServiceImpl } from 'vs/editor/common/services/modelServiceImpl';
 import { ITextResourcePropertiesService } from 'vs/editor/common/services/textResourceConfigurationService';
 import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
 import { TestConfigurationService } from 'vs/platform/configuration/test/common/testConfigurationService';
-import { TestThemeService } from 'vs/platform/theme/test/common/testThemeService';
+import { TestColorTheme, TestThemeService } from 'vs/platform/theme/test/common/testThemeService';
 import { NullLogService } from 'vs/platform/log/common/log';
 import { UndoRedoService } from 'vs/platform/undoRedo/common/undoRedoService';
 import { TestDialogService } from 'vs/platform/dialogs/test/common/testDialogService';
 import { TestNotificationService } from 'vs/platform/notification/test/common/testNotificationService';
 import { createTextModel } from 'vs/editor/test/common/editorTestUtils';
+import { DisposableStore } from 'vs/base/common/lifecycle';
+import { DocumentSemanticTokensProvider, DocumentSemanticTokensProviderRegistry, SemanticTokens, SemanticTokensEdits, SemanticTokensLegend } from 'vs/editor/common/modes';
+import { CancellationToken } from 'vs/base/common/cancellation';
+import { Barrier, timeout } from 'vs/base/common/async';
+import { ModeServiceImpl } from 'vs/editor/common/services/modeServiceImpl';
+import { ColorScheme } from 'vs/platform/theme/common/theme';
+import { ModesRegistry } from 'vs/editor/common/modes/modesRegistry';
+import { IModelService } from 'vs/editor/common/services/modelService';
+import { IModeService } from 'vs/editor/common/services/modeService';
 
 const GENERATE_TESTS = false;
 
@@ -378,6 +387,87 @@ suite('ModelService', () => {
 	});
 });
 
+suite('ModelSemanticColoring', () => {
+
+	const disposables = new DisposableStore();
+	const ORIGINAL_FETCH_DOCUMENT_SEMANTIC_TOKENS_DELAY = ModelSemanticColoring.FETCH_DOCUMENT_SEMANTIC_TOKENS_DELAY;
+	let modelService: IModelService;
+	let modeService: IModeService;
+
+	setup(() => {
+		ModelSemanticColoring.FETCH_DOCUMENT_SEMANTIC_TOKENS_DELAY = 0;
+
+		const configService = new TestConfigurationService({ editor: { semanticHighlighting: true } });
+		const themeService = new TestThemeService();
+		themeService.setTheme(new TestColorTheme({}, ColorScheme.DARK, true));
+		modelService = disposables.add(new ModelServiceImpl(
+			configService,
+			new TestTextResourcePropertiesService(configService),
+			themeService,
+			new NullLogService(),
+			new UndoRedoService(new TestDialogService(), new TestNotificationService())
+		));
+		modeService = disposables.add(new ModeServiceImpl(false));
+	});
+
+	teardown(() => {
+		disposables.clear();
+		ModelSemanticColoring.FETCH_DOCUMENT_SEMANTIC_TOKENS_DELAY = ORIGINAL_FETCH_DOCUMENT_SEMANTIC_TOKENS_DELAY;
+	});
+
+	test('DocumentSemanticTokens should be fetched when the result is empty if there are pending changes', async () => {
+
+		disposables.add(ModesRegistry.registerLanguage({ id: 'testMode' }));
+
+		const inFirstCall = new Barrier();
+		const delayFirstResult = new Barrier();
+		const secondResultProvided = new Barrier();
+		let callCount = 0;
+
+		disposables.add(DocumentSemanticTokensProviderRegistry.register('testMode', new class implements DocumentSemanticTokensProvider {
+			getLegend(): SemanticTokensLegend {
+				return { tokenTypes: ['class'], tokenModifiers: [] };
+			}
+			async provideDocumentSemanticTokens(model: ITextModel, lastResultId: string | null, token: CancellationToken): Promise<SemanticTokens | SemanticTokensEdits | null> {
+				callCount++;
+				if (callCount === 1) {
+					assert.ok('called once');
+					inFirstCall.open();
+					await delayFirstResult.wait();
+					await timeout(0); // wait for the simple scheduler to fire to check that we do actually get rescheduled
+					return null;
+				}
+				if (callCount === 2) {
+					assert.ok('called twice');
+					secondResultProvided.open();
+					return null;
+				}
+				assert.fail('Unexpected call');
+			}
+			releaseDocumentSemanticTokens(resultId: string | undefined): void {
+			}
+		}));
+
+		const textModel = disposables.add(modelService.createModel('Hello world', modeService.create('testMode')));
+
+		// wait for the provider to be called
+		await inFirstCall.wait();
+
+		// the provider is now in the provide call
+		// change the text buffer while the provider is running
+		textModel.applyEdits([{ range: new Range(1, 1, 1, 1), text: 'x' }]);
+
+		// let the provider finish its first result
+		delayFirstResult.open();
+
+		// we need to check that the provider is called again, even if it returns null
+		await secondResultProvided.wait();
+
+		// assert that it got called twice
+		assert.strictEqual(callCount, 2);
+	});
+});
+
 function assertComputeEdits(lines1: string[], lines2: string[]): void {
 	const model = createTextModel(lines1.join('\n'));
 	const textBuffer = createTextBuffer(lines2.join('\n'), DefaultEndOfLine.LF).textBuffer;

src/vs/platform/theme/test/common/testThemeService.ts

@@ -12,8 +12,11 @@ export class TestColorTheme implements IColorTheme {
 
 	public readonly label = 'test';
 
-	constructor(private colors: { [id: string]: string; } = {}, public type = ColorScheme.DARK) {
-	}
+	constructor(
+		private colors: { [id: string]: string; } = {},
+		public type = ColorScheme.DARK,
+		public readonly semanticHighlighting = false
+	) { }
 
 	getColor(color: string, useDefault?: boolean): Color | undefined {
 		let value = this.colors[color];
@@ -31,8 +34,6 @@ export class TestColorTheme implements IColorTheme {
 		return undefined;
 	}
 
-	readonly semanticHighlighting = false;
-
 	get tokenColorMap(): string[] {
 		return [];
 	}