Make the grammars extension point dynamic

src/vs/editor/common/modes.ts

@@ -1405,7 +1405,7 @@ export interface ITokenizationRegistry {
 	/**
 	 * Register a promise for a tokenization support.
 	 */
-	registerPromise(language: string, promise: Thenable<ITokenizationSupport>): Thenable<IDisposable>;
+	registerPromise(language: string, promise: Thenable<ITokenizationSupport>): IDisposable;
 
 	/**
 	 * Get the tokenization support for a language.

src/vs/editor/common/modes/tokenizationRegistry.ts

@@ -5,13 +5,13 @@
 
 import { Color } from 'vs/base/common/color';
 import { Emitter, Event } from 'vs/base/common/event';
-import { IDisposable, toDisposable, Disposable } from 'vs/base/common/lifecycle';
+import { IDisposable, toDisposable } from 'vs/base/common/lifecycle';
 import { ColorId, ITokenizationRegistry, ITokenizationSupport, ITokenizationSupportChangedEvent } from 'vs/editor/common/modes';
 
 export class TokenizationRegistryImpl implements ITokenizationRegistry {
 
 	private _map: { [language: string]: ITokenizationSupport };
-	private _promises: { [language: string]: Thenable<IDisposable> };
+	private _promises: { [language: string]: Thenable<void> };
 
 	private readonly _onDidChange: Emitter<ITokenizationSupportChangedEvent> = new Emitter<ITokenizationSupportChangedEvent>();
 	public readonly onDidChange: Event<ITokenizationSupportChangedEvent> = this._onDidChange.event;
@@ -43,16 +43,25 @@ export class TokenizationRegistryImpl implements ITokenizationRegistry {
 		});
 	}
 
-	public registerPromise(language: string, supportPromise: Thenable<ITokenizationSupport | null>): Thenable<IDisposable> {
-		const promise = this._promises[language] = supportPromise.then(support => {
+	public registerPromise(language: string, supportPromise: Thenable<ITokenizationSupport | null>): IDisposable {
+
+		let registration: IDisposable | null = null;
+		let isDisposed: boolean = false;
+
+		this._promises[language] = supportPromise.then(support => {
 			delete this._promises[language];
-			if (support) {
-				return this.register(language, support);
-			} else {
-				return Disposable.None;
+			if (isDisposed || !support) {
+				return;
+			}
+			registration = this.register(language, support);
+		});
+
+		return toDisposable(() => {
+			isDisposed = true;
+			if (registration) {
+				registration.dispose();
 			}
 		});
-		return promise;
 	}
 
 	public getPromise(language: string): Thenable<ITokenizationSupport> | null {

src/vs/workbench/services/textMate/electron-browser/TMGrammars.ts

@@ -25,6 +25,7 @@ export interface ITMSyntaxExtensionPoint {
 }
 
 export const grammarsExtPoint: IExtensionPoint<ITMSyntaxExtensionPoint[]> = ExtensionsRegistry.registerExtensionPoint<ITMSyntaxExtensionPoint[]>({
+	isDynamic: true,
 	extensionPoint: 'grammars',
 	deps: [languagesExtPoint],
 	jsonSchema: {

src/vs/workbench/services/textMate/electron-browser/TMSyntax.ts

@@ -19,13 +19,12 @@ import { IModeService } from 'vs/editor/common/services/modeService';
 import { IFileService } from 'vs/platform/files/common/files';
 import { ILogService } from 'vs/platform/log/common/log';
 import { INotificationService } from 'vs/platform/notification/common/notification';
-import { IExtensionService } from 'vs/workbench/services/extensions/common/extensions';
 import { ExtensionMessageCollector } from 'vs/workbench/services/extensions/common/extensionsRegistry';
 import { IEmbeddedLanguagesMap, ITMSyntaxExtensionPoint, TokenTypesContribution, grammarsExtPoint } from 'vs/workbench/services/textMate/electron-browser/TMGrammars';
 import { ITextMateService } from 'vs/workbench/services/textMate/electron-browser/textMateService';
 import { ITokenColorizationRule, IWorkbenchThemeService } from 'vs/workbench/services/themes/common/workbenchThemeService';
 import { IEmbeddedLanguagesMap as IEmbeddedLanguagesMap2, IGrammar, ITokenTypeMap, Registry, StackElement, StandardTokenType } from 'vscode-textmate';
-import { Disposable } from 'vs/base/common/lifecycle';
+import { Disposable, IDisposable, dispose } from 'vs/base/common/lifecycle';
 
 export class TMScopeRegistry {
 
@@ -36,6 +35,10 @@ export class TMScopeRegistry {
 	public readonly onDidEncounterLanguage: Event<LanguageId> = this._onDidEncounterLanguage.event;
 
 	constructor() {
+		this.reset();
+	}
+
+	public reset(): void {
 		this._scopeNameToLanguageRegistration = Object.create(null);
 		this._encounteredLanguages = [];
 	}
@@ -140,41 +143,55 @@ export class TextMateService extends Disposable implements ITextMateService {
 	public readonly onDidEncounterLanguage: Event<LanguageId> = this._onDidEncounterLanguage.event;
 
 	private readonly _styleElement: HTMLStyleElement;
+	private readonly _createdModes: string[];
 
 	private _scopeRegistry: TMScopeRegistry;
 	private _injections: { [scopeName: string]: string[]; };
 	private _injectedEmbeddedLanguages: { [scopeName: string]: IEmbeddedLanguagesMap[]; };
 	private _languageToScope: Map<string, string>;
 	private _grammarRegistry: Promise<[Registry, StackElement]> | null;
+	private _tokenizersRegistrations: IDisposable[];
 
-	private _currentTokenColors: ITokenColorizationRule[];
+	private _currentTokenColors: ITokenColorizationRule[] | null;
 
 	constructor(
 		@IModeService private readonly _modeService: IModeService,
 		@IWorkbenchThemeService private readonly _themeService: IWorkbenchThemeService,
 		@IFileService private readonly _fileService: IFileService,
 		@INotificationService private readonly _notificationService: INotificationService,
-		@ILogService private readonly _logService: ILogService,
-		@IExtensionService private readonly _extensionService: IExtensionService
+		@ILogService private readonly _logService: ILogService
 	) {
 		super();
 		this._styleElement = dom.createStyleSheet();
 		this._styleElement.className = 'vscode-tokens-styles';
+		this._createdModes = [];
 		this._scopeRegistry = new TMScopeRegistry();
 		this._scopeRegistry.onDidEncounterLanguage((language) => this._onDidEncounterLanguage.fire(language));
 		this._injections = {};
 		this._injectedEmbeddedLanguages = {};
 		this._languageToScope = new Map<string, string>();
 		this._grammarRegistry = null;
+		this._tokenizersRegistrations = [];
 		this._currentTokenColors = null;
 
 		grammarsExtPoint.setHandler((extensions) => {
+			this._scopeRegistry.reset();
+			this._injections = {};
+			this._injectedEmbeddedLanguages = {};
+			this._languageToScope = new Map<string, string>();
+			this._grammarRegistry = null;
+			this._tokenizersRegistrations = dispose(this._tokenizersRegistrations);
+
 			for (let i = 0; i < extensions.length; i++) {
 				let grammars = extensions[i].value;
 				for (let j = 0; j < grammars.length; j++) {
 					this._handleGrammarExtensionPointUser(extensions[i].description.extensionLocation, grammars[j], extensions[i].collector);
 				}
 			}
+
+			for (let i = 0; i < this._createdModes.length; i++) {
+				this._registerDefinitionIfAvailable(this._createdModes[i]);
+			}
 		});
 
 		// Generate some color map until the grammar registry is loaded
@@ -196,13 +213,23 @@ export class TextMateService extends Disposable implements ITextMateService {
 
 		this._modeService.onDidCreateMode((mode) => {
 			let modeId = mode.getId();
-			// Modes can be instantiated before the extension points have finished registering
-			this._extensionService.whenInstalledExtensionsRegistered().then(() => {
-				this._registerDefinitionIfAvailable(modeId);
-			});
+			this._createdModes.push(modeId);
+			this._registerDefinitionIfAvailable(modeId);
 		});
 	}
 
+	private _registerDefinitionIfAvailable(modeId: string): void {
+		if (this._languageToScope.has(modeId)) {
+			const promise = this._createGrammar(modeId).then((r) => {
+				return new TMTokenization(this._scopeRegistry, r.languageId, r.grammar, r.initialState, r.containsEmbeddedLanguages, this._notificationService);
+			}, e => {
+				onUnexpectedError(e);
+				return null;
+			});
+			this._tokenizersRegistrations.push(TokenizationRegistry.registerPromise(modeId, promise));
+		}
+	}
+
 	private _getOrCreateGrammarRegistry(): Promise<[Registry, StackElement]> {
 		if (!this._grammarRegistry) {
 			this._grammarRegistry = import('vscode-textmate').then(({ Registry, INITIAL, parseRawGrammar }) => {
@@ -284,7 +311,6 @@ export class TextMateService extends Disposable implements ITextMateService {
 		return false;
 	}
 
-
 	private _handleGrammarExtensionPointUser(extensionLocation: URI, syntax: ITMSyntaxExtensionPoint, collector: ExtensionMessageCollector): void {
 		if (syntax.language && ((typeof syntax.language !== 'string') || !this._modeService.isRegisteredMode(syntax.language))) {
 			collector.error(nls.localize('invalid.language', "Unknown language in `contributes.{0}.language`. Provided value: {1}", grammarsExtPoint.name, String(syntax.language)));
@@ -395,18 +421,6 @@ export class TextMateService extends Disposable implements ITextMateService {
 			});
 		});
 	}
-
-	private _registerDefinitionIfAvailable(modeId: string): void {
-		if (this._languageToScope.has(modeId)) {
-			const promise = this._createGrammar(modeId).then((r) => {
-				return new TMTokenization(this._scopeRegistry, r.languageId, r.grammar, r.initialState, r.containsEmbeddedLanguages, this._notificationService);
-			}, e => {
-				onUnexpectedError(e);
-				return null;
-			});
-			TokenizationRegistry.registerPromise(modeId, promise);
-		}
-	}
 }
 
 class TMTokenization implements ITokenizationSupport {