Allow to register a tokenizer promise and honor tokenizer promises when...

Allow to register a tokenizer promise and honor tokenizer promises when dealing with embedded modes in colorizing API

src/vs/editor/standalone/browser/colorizer.ts

@@ -13,6 +13,7 @@ import { IModeService } from 'vs/editor/common/services/modeService';
 import { RenderLineInput, renderViewLine2 as renderViewLine } from 'vs/editor/common/viewLayout/viewLineRenderer';
 import { ViewLineRenderingData } from 'vs/editor/common/viewModel/viewModel';
 import { IStandaloneThemeService } from 'vs/editor/standalone/common/standaloneThemeService';
+import { MonarchTokenizer } from 'vs/editor/standalone/common/monarch/monarchLexer';
 
 export interface IColorizerOptions {
 	tabSize?: number;
@@ -82,9 +83,10 @@ export class Colorizer {
 				}
 				const tokenizationSupport = TokenizationRegistry.get(language!);
 				if (tokenizationSupport) {
-					return resolve(_colorize(lines, tabSize, tokenizationSupport));
+					_colorize(lines, tabSize, tokenizationSupport).then(resolve, reject);
+					return;
 				}
-				return resolve(_fakeColorize(lines, tabSize));
+				resolve(_fakeColorize(lines, tabSize));
 			};
 
 			// wait 500ms for mode to load, then give up
@@ -130,8 +132,21 @@ export class Colorizer {
 	}
 }
 
-function _colorize(lines: string[], tabSize: number, tokenizationSupport: ITokenizationSupport): string {
-	return _actualColorize(lines, tabSize, tokenizationSupport);
+function _colorize(lines: string[], tabSize: number, tokenizationSupport: ITokenizationSupport): Promise<string> {
+	return new Promise<string>((c, e) => {
+		const execute = () => {
+			const result = _actualColorize(lines, tabSize, tokenizationSupport);
+			if (tokenizationSupport instanceof MonarchTokenizer) {
+				const status = tokenizationSupport.getLoadStatus();
+				if (status.loaded === false) {
+					status.promise.then(execute, e);
+					return;
+				}
+			}
+			c(result);
+		};
+		execute();
+	});
 }
 
 function _fakeColorize(lines: string[], tabSize: number): string {

src/vs/editor/standalone/browser/standaloneLanguages.ts

@@ -292,31 +292,47 @@ export interface EncodedTokensProvider {
 function isEncodedTokensProvider(provider: TokensProvider | EncodedTokensProvider): provider is EncodedTokensProvider {
 	return provider['tokenizeEncoded'];
 }
+
+function isThenable<T>(obj: any): obj is Thenable<T> {
+	if (typeof obj.then === 'function') {
+		return true;
+	}
+	return false;
+}
+
 /**
  * Set the tokens provider for a language (manual implementation).
  */
-export function setTokensProvider(languageId: string, provider: TokensProvider | EncodedTokensProvider): IDisposable {
+export function setTokensProvider(languageId: string, provider: TokensProvider | EncodedTokensProvider | Thenable<TokensProvider | EncodedTokensProvider>): IDisposable {
 	let languageIdentifier = StaticServices.modeService.get().getLanguageIdentifier(languageId);
 	if (!languageIdentifier) {
 		throw new Error(`Cannot set tokens provider for unknown language ${languageId}`);
 	}
-	let adapter: modes.ITokenizationSupport;
-	if (isEncodedTokensProvider(provider)) {
-		adapter = new EncodedTokenizationSupport2Adapter(provider);
-	} else {
-		adapter = new TokenizationSupport2Adapter(StaticServices.standaloneThemeService.get(), languageIdentifier, provider);
+	const create = (provider: TokensProvider | EncodedTokensProvider) => {
+		if (isEncodedTokensProvider(provider)) {
+			return new EncodedTokenizationSupport2Adapter(provider);
+		} else {
+			return new TokenizationSupport2Adapter(StaticServices.standaloneThemeService.get(), languageIdentifier!, provider);
+		}
+	};
+	if (isThenable<TokensProvider | EncodedTokensProvider>(provider)) {
+		return modes.TokenizationRegistry.registerPromise(languageId, provider.then(provider => create(provider)));
 	}
-	return modes.TokenizationRegistry.register(languageId, adapter);
+	return modes.TokenizationRegistry.register(languageId, create(provider));
 }
 
 
 /**
  * Set the tokens provider for a language (monarch implementation).
  */
-export function setMonarchTokensProvider(languageId: string, languageDef: IMonarchLanguage): IDisposable {
-	let lexer = compile(languageId, languageDef);
-	let adapter = createTokenizationSupport(StaticServices.modeService.get(), StaticServices.standaloneThemeService.get(), languageId, lexer);
-	return modes.TokenizationRegistry.register(languageId, adapter);
+export function setMonarchTokensProvider(languageId: string, languageDef: IMonarchLanguage | Thenable<IMonarchLanguage>): IDisposable {
+	const create = (languageDef: IMonarchLanguage) => {
+		return createTokenizationSupport(StaticServices.modeService.get(), StaticServices.standaloneThemeService.get(), languageId, compile(languageId, languageDef));
+	};
+	if (isThenable<IMonarchLanguage>(languageDef)) {
+		return modes.TokenizationRegistry.registerPromise(languageId, languageDef.then(languageDef => create(languageDef)));
+	}
+	return modes.TokenizationRegistry.register(languageId, create(languageDef));
 }
 
 /**

src/vs/editor/standalone/common/monarch/monarchLexer.ts

@@ -374,13 +374,16 @@ class MonarchModernTokensCollector implements IMonarchTokensCollector {
 	}
 }
 
-class MonarchTokenizer implements modes.ITokenizationSupport {
+export type ILoadStatus = { loaded: true; } | { loaded: false; promise: Promise<void>; };
+
+export class MonarchTokenizer implements modes.ITokenizationSupport {
 
 	private readonly _modeService: IModeService;
 	private readonly _standaloneThemeService: IStandaloneThemeService;
 	private readonly _modeId: string;
 	private readonly _lexer: monarchCommon.ILexer;
 	private _embeddedModes: { [modeId: string]: boolean; };
+	public embeddedLoaded: Promise<void>;
 	private _tokenizationRegistryListener: IDisposable;
 
 	constructor(modeService: IModeService, standaloneThemeService: IStandaloneThemeService, modeId: string, lexer: monarchCommon.ILexer) {
@@ -389,6 +392,7 @@ class MonarchTokenizer implements modes.ITokenizationSupport {
 		this._modeId = modeId;
 		this._lexer = lexer;
 		this._embeddedModes = Object.create(null);
+		this.embeddedLoaded = Promise.resolve(undefined);
 
 		// Set up listening for embedded modes
 		let emitting = false;
@@ -416,6 +420,39 @@ class MonarchTokenizer implements modes.ITokenizationSupport {
 		this._tokenizationRegistryListener.dispose();
 	}
 
+	public getLoadStatus(): ILoadStatus {
+		let promises: Thenable<any>[] = [];
+		for (let nestedModeId in this._embeddedModes) {
+			const tokenizationSupport = modes.TokenizationRegistry.get(nestedModeId);
+			if (tokenizationSupport) {
+				// The nested mode is already loaded
+				if (tokenizationSupport instanceof MonarchTokenizer) {
+					const nestedModeStatus = tokenizationSupport.getLoadStatus();
+					if (nestedModeStatus.loaded === false) {
+						promises.push(nestedModeStatus.promise);
+					}
+				}
+				continue;
+			}
+
+			const tokenizationSupportPromise = modes.TokenizationRegistry.getPromise(nestedModeId);
+			if (tokenizationSupportPromise) {
+				// The nested mode is in the process of being loaded
+				promises.push(tokenizationSupportPromise);
+			}
+		}
+
+		if (promises.length === 0) {
+			return {
+				loaded: true
+			};
+		}
+		return {
+			loaded: false,
+			promise: Promise.all(promises).then(_ => undefined)
+		};
+	}
+
 	public getInitialState(): modes.IState {
 		let rootState = MonarchStackElementFactory.create(null, this._lexer.start!);
 		return MonarchLineStateFactory.create(rootState, null);

src/vs/monaco.d.ts

@@ -4284,12 +4284,12 @@ declare namespace monaco.languages {
 	/**
 	 * Set the tokens provider for a language (manual implementation).
 	 */
-	export function setTokensProvider(languageId: string, provider: TokensProvider | EncodedTokensProvider): IDisposable;
+	export function setTokensProvider(languageId: string, provider: TokensProvider | EncodedTokensProvider | Thenable<TokensProvider | EncodedTokensProvider>): IDisposable;
 
 	/**
 	 * Set the tokens provider for a language (monarch implementation).
 	 */
-	export function setMonarchTokensProvider(languageId: string, languageDef: IMonarchLanguage): IDisposable;
+	export function setMonarchTokensProvider(languageId: string, languageDef: IMonarchLanguage | Thenable<IMonarchLanguage>): IDisposable;
 
 	/**
 	 * Register a reference provider (used by e.g. reference search).