Fixes #14661: `embeddedLanguages` is now scoped to the contribution point...

Fixes #14661: `embeddedLanguages` is now scoped to the contribution point (i.e. it is not applied for all grammars)

src/vs/editor/node/textMate/TMSyntax.ts

@@ -20,11 +20,15 @@ import { ModeTransition } from 'vs/editor/common/core/modeTransition';
 import { Token } from 'vs/editor/common/core/token';
 import { languagesExtPoint } from 'vs/editor/common/services/modeServiceImpl';
 
+export interface IEmbeddedLanguagesMap {
+	[scopeName: string]: string;
+}
+
 export interface ITMSyntaxExtensionPoint {
 	language: string;
 	scopeName: string;
 	path: string;
-	embeddedLanguages: { [scopeName: string]: string; };
+	embeddedLanguages: IEmbeddedLanguagesMap;
 	injectTo: string[];
 }
 
@@ -67,55 +71,83 @@ export const grammarsExtPoint: IExtensionPoint<ITMSyntaxExtensionPoint[]> = Exte
 
 export class TMScopeRegistry {
 
-	private _scopeNameToFilePath: { [scopeName: string]: string; };
-	private _scopeNameToLanguage: { [scopeName: string]: string; };
+	private _scopeNameToLanguageRegistration: { [scopeName: string]: TMLanguageRegistration; };
 	private _encounteredLanguages: { [language: string]: boolean; };
-	private _cachedScopesRegex: RegExp;
 
 	private _onDidEncounterLanguage: Emitter<string> = new Emitter<string>();
 	public onDidEncounterLanguage: Event<string> = this._onDidEncounterLanguage.event;
 
 	constructor() {
-		this._scopeNameToFilePath = Object.create(null);
-		this._scopeNameToLanguage = Object.create(null);
+		this._scopeNameToLanguageRegistration = Object.create(null);
 		this._encounteredLanguages = Object.create(null);
-		this._cachedScopesRegex = null;
 	}
 
-	public register(language: string, scopeName: string, filePath: string): void {
-		this._scopeNameToFilePath[scopeName] = filePath;
+	public register(scopeName: string, filePath: string, embeddedLanguages?: IEmbeddedLanguagesMap): void {
+		this._scopeNameToLanguageRegistration[scopeName] = new TMLanguageRegistration(this, scopeName, filePath, embeddedLanguages);
 	}
 
-	public registerEmbeddedLanguages(scopeToLanguageMap: { [scopeName: string]: string }): void {
-		var scopes = Object.keys(scopeToLanguageMap);
-		for (let i = 0, len = scopes.length; i < len; i++) {
-			let scope = scopes[i];
-			let language = scopeToLanguageMap[scope];
-			if (typeof language !== 'string') {
-				// never hurts to be too careful
-				continue;
-			}
-			this._scopeNameToLanguage[scope] = language;
-			this._cachedScopesRegex = null;
-		}
+	public getLanguageRegistration(scopeName: string): TMLanguageRegistration {
+		return this._scopeNameToLanguageRegistration[scopeName] || null;
 	}
 
 	public getFilePath(scopeName: string): string {
-		return this._scopeNameToFilePath[scopeName] || null;
+		let data = this.getLanguageRegistration(scopeName);
+		return data ? data.grammarFilePath : null;
 	}
 
-	private _getScopesRegex(): RegExp {
-		if (!this._cachedScopesRegex) {
-			let escapedScopes = Object.keys(this._scopeNameToLanguage).map((scopeName) => strings.escapeRegExpCharacters(scopeName));
-			if (escapedScopes.length === 0) {
-				// no scopes registered
-				return null;
+	/**
+	 * To be called when tokenization found/hit an embedded language.
+	 */
+	public onEncounteredLanguage(language: string): void {
+		if (!this._encounteredLanguages[language]) {
+			this._encounteredLanguages[language] = true;
+			this._onDidEncounterLanguage.fire(language);
+		}
+	}
+}
+
+export class TMLanguageRegistration {
+	_topLevelScopeNameDataBrand: void;
+
+	readonly scopeName: string;
+	readonly grammarFilePath: string;
+
+	private readonly _registry: TMScopeRegistry;
+	private readonly _embeddedLanguages: IEmbeddedLanguagesMap;
+	private readonly _embeddedLanguagesRegex: RegExp;
+
+	constructor(registry: TMScopeRegistry, scopeName: string, grammarFilePath: string, embeddedLanguages: IEmbeddedLanguagesMap) {
+		this._registry = registry;
+		this.scopeName = scopeName;
+		this.grammarFilePath = grammarFilePath;
+
+		// embeddedLanguages handling
+		this._embeddedLanguages = Object.create(null);
+
+		if (embeddedLanguages) {
+			// If embeddedLanguages are configured, fill in `this._embeddedLanguages`
+			let scopes = Object.keys(embeddedLanguages);
+			for (let i = 0, len = scopes.length; i < len; i++) {
+				let scope = scopes[i];
+				let language = embeddedLanguages[scope];
+				if (typeof language !== 'string') {
+					// never hurts to be too careful
+					continue;
+				}
+				this._embeddedLanguages[scope] = language;
 			}
+		}
+
+		// create the regex
+		let escapedScopes = Object.keys(this._embeddedLanguages).map((scopeName) => strings.escapeRegExpCharacters(scopeName));
+		if (escapedScopes.length === 0) {
+			// no scopes registered
+			this._embeddedLanguagesRegex = null;
+		} else {
 			escapedScopes.sort();
 			escapedScopes.reverse();
-			this._cachedScopesRegex = new RegExp(`^((${escapedScopes.join(')|(')}))($|\\.)`, '');
+			this._embeddedLanguagesRegex = new RegExp(`^((${escapedScopes.join(')|(')}))($|\\.)`, '');
 		}
-		return this._cachedScopesRegex;
 	}
 
 	/**
@@ -126,23 +158,22 @@ export class TMScopeRegistry {
 		if (!scope) {
 			return null;
 		}
-		let regex = this._getScopesRegex();
-		if (!regex) {
+		if (!this._embeddedLanguagesRegex) {
 			// no scopes registered
 			return null;
 		}
-		let m = scope.match(regex);
+		let m = scope.match(this._embeddedLanguagesRegex);
 		if (!m) {
 			// no scopes matched
 			return null;
 		}
 
-		let language = this._scopeNameToLanguage[m[1]] || null;
-		if (language && !this._encounteredLanguages[language]) {
-			this._encounteredLanguages[language] = true;
-			this._onDidEncounterLanguage.fire(language);
+		let language = this._embeddedLanguages[m[1]] || null;
+		if (!language) {
+			return null;
 		}
 
+		this._registry.onEncounteredLanguage(language);
 		return language;
 	}
 }
@@ -210,11 +241,7 @@ export class MainProcessTextMateSyntax {
 			collector.warn(nls.localize('invalid.path.1', "Expected `contributes.{0}.path` ({1}) to be included inside extension's folder ({2}). This might make the extension non-portable.", grammarsExtPoint.name, normalizedAbsolutePath, extensionFolderPath));
 		}
 
-		this._scopeRegistry.register(syntax.language, syntax.scopeName, normalizedAbsolutePath);
-
-		if (syntax.embeddedLanguages) {
-			this._scopeRegistry.registerEmbeddedLanguages(syntax.embeddedLanguages);
-		}
+		this._scopeRegistry.register(syntax.scopeName, normalizedAbsolutePath, syntax.embeddedLanguages);
 
 		if (syntax.injectTo) {
 			for (let injectScope of syntax.injectTo) {
@@ -245,13 +272,14 @@ export class MainProcessTextMateSyntax {
 				return;
 			}
 
-			TokenizationRegistry.register(modeId, createTokenizationSupport(this._scopeRegistry, scopeName, modeId, grammar));
+			let languageRegistration = this._scopeRegistry.getLanguageRegistration(scopeName);
+			TokenizationRegistry.register(modeId, createTokenizationSupport(languageRegistration, modeId, grammar));
 		});
 	}
 }
 
-function createTokenizationSupport(scopeRegistry: TMScopeRegistry, topLevelScopeName: string, modeId: string, grammar: IGrammar): ITokenizationSupport {
-	var tokenizer = new Tokenizer(scopeRegistry, topLevelScopeName, modeId, grammar);
+function createTokenizationSupport(languageRegistration: TMLanguageRegistration, modeId: string, grammar: IGrammar): ITokenizationSupport {
+	var tokenizer = new Tokenizer(languageRegistration, modeId, grammar);
 	return {
 		getInitialState: () => new TMState(modeId, null, null),
 		tokenize: (line, state, offsetDelta?, stopAtOffset?) => tokenizer.tokenize(line, <TMState>state, offsetDelta, stopAtOffset)
@@ -343,21 +371,21 @@ export class DecodeMap {
 	_decodeMapBrand: void;
 
 	private lastAssignedTokenId: number;
-	private scopeRegistry: TMScopeRegistry;
+	private readonly languageRegistration: TMLanguageRegistration;
 	private readonly scopeToTokenIds: { [scope: string]: TMScopeDecodeData; };
 	private readonly tokenToTokenId: { [token: string]: number; };
 	private readonly tokenIdToToken: string[];
 	prevTokenScopes: TMScopesDecodeData[];
 	public readonly topLevelScope: TMScopesDecodeData;
 
-	constructor(scopeRegistry: TMScopeRegistry, topLevelScopeName: string) {
+	constructor(languageRegistration: TMLanguageRegistration) {
 		this.lastAssignedTokenId = 0;
-		this.scopeRegistry = scopeRegistry;
+		this.languageRegistration = languageRegistration;
 		this.scopeToTokenIds = Object.create(null);
 		this.tokenToTokenId = Object.create(null);
 		this.tokenIdToToken = [null];
 		this.prevTokenScopes = [];
-		this.topLevelScope = new TMScopesDecodeData(null, new TMScopeDecodeData(topLevelScopeName, this.scopeRegistry.scopeToLanguage(topLevelScopeName), []));
+		this.topLevelScope = new TMScopesDecodeData(null, new TMScopeDecodeData(languageRegistration.scopeName, this.languageRegistration.scopeToLanguage(languageRegistration.scopeName), []));
 	}
 
 	private _getTokenId(token: string): number {
@@ -383,7 +411,7 @@ export class DecodeMap {
 			tokenIds[i] = this._getTokenId(scopePieces[i]);
 		}
 
-		result = new TMScopeDecodeData(scope, this.scopeRegistry.scopeToLanguage(scope), tokenIds);
+		result = new TMScopeDecodeData(scope, this.languageRegistration.scopeToLanguage(scope), tokenIds);
 		this.scopeToTokenIds[scope] = result;
 		return result;
 	}
@@ -420,10 +448,10 @@ class Tokenizer {
 	private _modeId: string;
 	private _decodeMap: DecodeMap;
 
-	constructor(scopeRegistry: TMScopeRegistry, topLevelScopeName: string, modeId: string, grammar: IGrammar) {
+	constructor(languageRegistration: TMLanguageRegistration, modeId: string, grammar: IGrammar) {
 		this._modeId = modeId;
 		this._grammar = grammar;
-		this._decodeMap = new DecodeMap(scopeRegistry, topLevelScopeName);
+		this._decodeMap = new DecodeMap(languageRegistration);
 	}
 
 	public tokenize(line: string, state: TMState, offsetDelta: number = 0, stopAtOffset?: number): ILineTokens {

src/vs/editor/test/node/textMate/TMSyntax.test.ts

@@ -5,7 +5,7 @@
 'use strict';
 
 import * as assert from 'assert';
-import { decodeTextMateToken, decodeTextMateTokens, DecodeMap, TMScopeRegistry } from 'vs/editor/node/textMate/TMSyntax';
+import { decodeTextMateToken, decodeTextMateTokens, DecodeMap, TMScopeRegistry, TMLanguageRegistration } from 'vs/editor/node/textMate/TMSyntax';
 import { TMState } from 'vs/editor/common/modes/TMState';
 
 suite('TextMate.TMScopeRegistry', () => {
@@ -13,19 +13,19 @@ suite('TextMate.TMScopeRegistry', () => {
 	test('getFilePath', () => {
 		let registry = new TMScopeRegistry();
 
-		registry.register('a', 'source.a', './grammar/a.tmLanguage');
+		registry.register('source.a', './grammar/a.tmLanguage');
 		assert.equal(registry.getFilePath('source.a'), './grammar/a.tmLanguage');
 		assert.equal(registry.getFilePath('a'), null);
 		assert.equal(registry.getFilePath('source.b'), null);
 		assert.equal(registry.getFilePath('b'), null);
 
-		registry.register('b', 'source.b', './grammar/b.tmLanguage');
+		registry.register('source.b', './grammar/b.tmLanguage');
 		assert.equal(registry.getFilePath('source.a'), './grammar/a.tmLanguage');
 		assert.equal(registry.getFilePath('a'), null);
 		assert.equal(registry.getFilePath('source.b'), './grammar/b.tmLanguage');
 		assert.equal(registry.getFilePath('b'), null);
 
-		registry.register('a', 'source.a', './grammar/ax.tmLanguage');
+		registry.register('source.a', './grammar/ax.tmLanguage');
 		assert.equal(registry.getFilePath('source.a'), './grammar/ax.tmLanguage');
 		assert.equal(registry.getFilePath('a'), null);
 		assert.equal(registry.getFilePath('source.b'), './grammar/b.tmLanguage');
@@ -34,10 +34,7 @@ suite('TextMate.TMScopeRegistry', () => {
 
 	test('scopeToLanguage', () => {
 		let registry = new TMScopeRegistry();
-
-		assert.equal(registry.scopeToLanguage('source.html'), null);
-
-		registry.registerEmbeddedLanguages({
+		registry.register('source.html', './grammar/html.tmLanguage', {
 			'source.html': 'html',
 			'source.c': 'c',
 			'source.css': 'css',
@@ -46,29 +43,30 @@ suite('TextMate.TMScopeRegistry', () => {
 			'source.smarty': 'smarty',
 			'source.baz': null,
 		});
+		let languageRegistration = registry.getLanguageRegistration('source.html');
 
 		// exact matches
-		assert.equal(registry.scopeToLanguage('source.html'), 'html');
-		assert.equal(registry.scopeToLanguage('source.css'), 'css');
-		assert.equal(registry.scopeToLanguage('source.c'), 'c');
-		assert.equal(registry.scopeToLanguage('source.js'), 'javascript');
-		assert.equal(registry.scopeToLanguage('source.python'), 'python');
-		assert.equal(registry.scopeToLanguage('source.smarty'), 'smarty');
+		assert.equal(languageRegistration.scopeToLanguage('source.html'), 'html');
+		assert.equal(languageRegistration.scopeToLanguage('source.css'), 'css');
+		assert.equal(languageRegistration.scopeToLanguage('source.c'), 'c');
+		assert.equal(languageRegistration.scopeToLanguage('source.js'), 'javascript');
+		assert.equal(languageRegistration.scopeToLanguage('source.python'), 'python');
+		assert.equal(languageRegistration.scopeToLanguage('source.smarty'), 'smarty');
 
 		// prefix matches
-		assert.equal(registry.scopeToLanguage('source.css.embedded.html'), 'css');
-		assert.equal(registry.scopeToLanguage('source.js.embedded.html'), 'javascript');
-		assert.equal(registry.scopeToLanguage('source.python.embedded.html'), 'python');
-		assert.equal(registry.scopeToLanguage('source.smarty.embedded.html'), 'smarty');
+		assert.equal(languageRegistration.scopeToLanguage('source.css.embedded.html'), 'css');
+		assert.equal(languageRegistration.scopeToLanguage('source.js.embedded.html'), 'javascript');
+		assert.equal(languageRegistration.scopeToLanguage('source.python.embedded.html'), 'python');
+		assert.equal(languageRegistration.scopeToLanguage('source.smarty.embedded.html'), 'smarty');
 
 		// misses
-		assert.equal(registry.scopeToLanguage('source.ts'), null);
-		assert.equal(registry.scopeToLanguage('source.csss'), null);
-		assert.equal(registry.scopeToLanguage('source.baz'), null);
-		assert.equal(registry.scopeToLanguage('asource.css'), null);
-		assert.equal(registry.scopeToLanguage('a.source.css'), null);
-		assert.equal(registry.scopeToLanguage('source_css'), null);
-		assert.equal(registry.scopeToLanguage('punctuation.definition.tag.html'), null);
+		assert.equal(languageRegistration.scopeToLanguage('source.ts'), null);
+		assert.equal(languageRegistration.scopeToLanguage('source.csss'), null);
+		assert.equal(languageRegistration.scopeToLanguage('source.baz'), null);
+		assert.equal(languageRegistration.scopeToLanguage('asource.css'), null);
+		assert.equal(languageRegistration.scopeToLanguage('a.source.css'), null);
+		assert.equal(languageRegistration.scopeToLanguage('source_css'), null);
+		assert.equal(languageRegistration.scopeToLanguage('punctuation.definition.tag.html'), null);
 	});
 
 });
@@ -77,8 +75,7 @@ suite('TextMate.decodeTextMateTokens', () => {
 
 	test('embedded modes', () => {
 		let registry = new TMScopeRegistry();
-
-		registry.registerEmbeddedLanguages({
+		registry.register('source.html', './grammar/html.tmLanguage', {
 			'source.html': 'html',
 			'source.c': 'c',
 			'source.css': 'css',
@@ -87,8 +84,9 @@ suite('TextMate.decodeTextMateTokens', () => {
 			'source.smarty': 'smarty',
 			'source.baz': null,
 		});
+		let languageRegistration = registry.getLanguageRegistration('source.html');
 
-		let decodeMap = new DecodeMap(registry, 'source.html');
+		let decodeMap = new DecodeMap(languageRegistration);
 		let actual = decodeTextMateTokens(
 			'text<style>body{}</style><script>var x=3;</script>text',
 			0,
@@ -377,7 +375,7 @@ suite('TextMate.decodeTextMateTokens', () => {
 
 		let registry = new TMScopeRegistry();
 
-		registry.registerEmbeddedLanguages({
+		registry.register('text.html.php', null, {
 			'text.html': 'html',
 			'source.php': 'php',
 			'source.sql': 'sql',
@@ -387,7 +385,7 @@ suite('TextMate.decodeTextMateTokens', () => {
 			'source.css': 'css'
 		});
 
-		let decodeMap = new DecodeMap(registry, 'text.html.php');
+		let decodeMap = new DecodeMap(registry.getLanguageRegistration('text.html.php'));
 
 		for (let i = 0, len = tests.length; i < len; i++) {
 			let test = tests[i];
@@ -818,7 +816,7 @@ suite('TextMate.decodeTextMateTokens', () => {
 
 		let registry = new TMScopeRegistry();
 
-		registry.registerEmbeddedLanguages({
+		registry.register('text.html.basic', null, {
 			'text.html.basic': 'html',
 			'source.css': 'css',
 			'source.js': 'javascript',
@@ -826,7 +824,7 @@ suite('TextMate.decodeTextMateTokens', () => {
 			'source.smarty': 'smarty'
 		});
 
-		let decodeMap = new DecodeMap(registry, 'text.html.basic');
+		let decodeMap = new DecodeMap(registry.getLanguageRegistration('text.html.basic'));
 
 		for (let i = 0, len = tests.length; i < len; i++) {
 			let test = tests[i];
@@ -839,6 +837,75 @@ suite('TextMate.decodeTextMateTokens', () => {
 			assert.deepEqual(actualModeTransitions, test.modeTransitions, 'test ' + test.line);
 		}
 	});
+
+	test('issue #14661: Comment shortcut in SCSS now using CSS style comments', () => {
+		let tests = [
+			{
+				line: 'class {',
+				tmTokens: [
+					{ startIndex: 0, endIndex: 6, scopes: [ 'source.css.scss' ] },
+					{ startIndex: 6, endIndex: 7, scopes: [ 'source.css.scss', 'meta.property-list.scss', 'punctuation.section.property-list.begin.bracket.curly.scss' ] }
+				],
+				tokens: [
+					{ startIndex: 0, type: '' },
+					{ startIndex: 6, type: 'meta.property-list.scss.punctuation.section.begin.bracket.curly' }
+				],
+				modeTransitions: [
+					{ startIndex: 0, modeId: 'scss' }
+				]
+			}, {
+				line: '    background: red;',
+				tmTokens: [
+					{ startIndex: 0, endIndex: 4, scopes: [ 'source.css.scss', 'meta.property-list.scss' ] },
+					{ startIndex: 4, endIndex: 14, scopes: [ 'source.css.scss', 'meta.property-list.scss', 'meta.property-name.scss', 'support.type.property-name.scss' ] },
+					{ startIndex: 14, endIndex: 15, scopes: [ 'source.css.scss', 'meta.property-list.scss', 'punctuation.separator.key-value.scss' ] },
+					{ startIndex: 15, endIndex: 16, scopes: [ 'source.css.scss', 'meta.property-list.scss' ] },
+					{ startIndex: 16, endIndex: 19, scopes: [ 'source.css.scss', 'meta.property-list.scss', 'meta.property-value.scss', 'support.constant.color.w3c-standard-color-name.scss' ] },
+					{ startIndex: 19, endIndex: 20, scopes: [ 'source.css.scss', 'meta.property-list.scss', 'punctuation.terminator.rule.scss' ] }
+				],
+				tokens: [
+					{ startIndex: 0, type: 'meta.property-list.scss' },
+					{ startIndex: 4, type: 'meta.property-list.scss.property-name.support.type' },
+					{ startIndex: 14, type: 'meta.property-list.scss.punctuation.separator.key-value' },
+					{ startIndex: 15, type: 'meta.property-list.scss' },
+					{ startIndex: 16, type: 'meta.property-list.scss.support.property-value.constant.color.w3c-standard-color-name' },
+					{ startIndex: 19, type: 'meta.property-list.scss.punctuation.terminator.rule' }
+				],
+				modeTransitions: [
+					{ startIndex: 0, modeId: 'scss' }
+				]
+			}, {
+				line: '}',
+				tmTokens: [
+					{ startIndex: 0, endIndex: 1, scopes: [ 'source.css.scss', 'meta.property-list.scss', 'punctuation.section.property-list.end.bracket.curly.scss' ] }
+				],
+				tokens: [
+					{ startIndex: 0, type: 'meta.property-list.scss.punctuation.section.bracket.curly.end' }
+				],
+				modeTransitions: [
+					{ startIndex: 0, modeId: 'scss' }
+				]
+			}
+		];
+
+		let registry = new TMScopeRegistry();
+
+		registry.register('source.css.scss', './syntaxes/scss.json');
+
+		let decodeMap = new DecodeMap(registry.getLanguageRegistration('source.css.scss'));
+
+		for (let i = 0, len = tests.length; i < len; i++) {
+			let test = tests[i];
+			let actual = decodeTextMateTokens(test.line, 0, decodeMap, test.tmTokens, new TMState('scss', null, null));
+
+			let actualTokens = actual.tokens.map((t) => { return { startIndex: t.startIndex, type: t.type }; });
+			let actualModeTransitions = actual.modeTransitions.map((t) => { return { startIndex: t.startIndex, modeId: t.modeId }; });
+
+			assert.deepEqual(actualTokens, test.tokens, 'test ' + test.line);
+			assert.deepEqual(actualModeTransitions, test.modeTransitions, 'test ' + test.line);
+		}
+
+	});
 });
 
 suite('textMate', () => {
@@ -874,7 +941,7 @@ suite('textMate', () => {
 	}
 
 	function testDecodeTextMateToken(input: string[][], expected: string[]): void {
-		let decodeMap = new DecodeMap(new TMScopeRegistry(), null);
+		let decodeMap = new DecodeMap(new TMLanguageRegistration(null, null, null, null));
 
 		for (let i = 0; i < input.length; i++) {
 			testOneDecodeTextMateToken(decodeMap, input[i], expected[i]);