diff --git a/src/vs/editor/node/textMate/TMSyntax.ts b/src/vs/editor/node/textMate/TMSyntax.ts index 99b84653116bdc8da066e466699b45aa31468152..deefd1590000419093c2370ff37516149c556400 100644 --- a/src/vs/editor/node/textMate/TMSyntax.ts +++ b/src/vs/editor/node/textMate/TMSyntax.ts @@ -20,11 +20,15 @@ import { ModeTransition } from 'vs/editor/common/core/modeTransition'; import { Token } from 'vs/editor/common/core/token'; import { languagesExtPoint } from 'vs/editor/common/services/modeServiceImpl'; +export interface IEmbeddedLanguagesMap { + [scopeName: string]: string; +} + export interface ITMSyntaxExtensionPoint { language: string; scopeName: string; path: string; - embeddedLanguages: { [scopeName: string]: string; }; + embeddedLanguages: IEmbeddedLanguagesMap; injectTo: string[]; } @@ -67,55 +71,83 @@ export const grammarsExtPoint: IExtensionPoint = Exte export class TMScopeRegistry { - private _scopeNameToFilePath: { [scopeName: string]: string; }; - private _scopeNameToLanguage: { [scopeName: string]: string; }; + private _scopeNameToLanguageRegistration: { [scopeName: string]: TMLanguageRegistration; }; private _encounteredLanguages: { [language: string]: boolean; }; - private _cachedScopesRegex: RegExp; private _onDidEncounterLanguage: Emitter = new Emitter(); public onDidEncounterLanguage: Event = this._onDidEncounterLanguage.event; constructor() { - this._scopeNameToFilePath = Object.create(null); - this._scopeNameToLanguage = Object.create(null); + this._scopeNameToLanguageRegistration = Object.create(null); this._encounteredLanguages = Object.create(null); - this._cachedScopesRegex = null; } - public register(language: string, scopeName: string, filePath: string): void { - this._scopeNameToFilePath[scopeName] = filePath; + public register(scopeName: string, filePath: string, embeddedLanguages?: IEmbeddedLanguagesMap): void { + this._scopeNameToLanguageRegistration[scopeName] = new TMLanguageRegistration(this, scopeName, filePath, embeddedLanguages); } - public registerEmbeddedLanguages(scopeToLanguageMap: { [scopeName: string]: string }): void { - var scopes = Object.keys(scopeToLanguageMap); - for (let i = 0, len = scopes.length; i < len; i++) { - let scope = scopes[i]; - let language = scopeToLanguageMap[scope]; - if (typeof language !== 'string') { - // never hurts to be too careful - continue; - } - this._scopeNameToLanguage[scope] = language; - this._cachedScopesRegex = null; - } + public getLanguageRegistration(scopeName: string): TMLanguageRegistration { + return this._scopeNameToLanguageRegistration[scopeName] || null; } public getFilePath(scopeName: string): string { - return this._scopeNameToFilePath[scopeName] || null; + let data = this.getLanguageRegistration(scopeName); + return data ? data.grammarFilePath : null; } - private _getScopesRegex(): RegExp { - if (!this._cachedScopesRegex) { - let escapedScopes = Object.keys(this._scopeNameToLanguage).map((scopeName) => strings.escapeRegExpCharacters(scopeName)); - if (escapedScopes.length === 0) { - // no scopes registered - return null; + /** + * To be called when tokenization found/hit an embedded language. + */ + public onEncounteredLanguage(language: string): void { + if (!this._encounteredLanguages[language]) { + this._encounteredLanguages[language] = true; + this._onDidEncounterLanguage.fire(language); + } + } +} + +export class TMLanguageRegistration { + _topLevelScopeNameDataBrand: void; + + readonly scopeName: string; + readonly grammarFilePath: string; + + private readonly _registry: TMScopeRegistry; + private readonly _embeddedLanguages: IEmbeddedLanguagesMap; + private readonly _embeddedLanguagesRegex: RegExp; + + constructor(registry: TMScopeRegistry, scopeName: string, grammarFilePath: string, embeddedLanguages: IEmbeddedLanguagesMap) { + this._registry = registry; + this.scopeName = scopeName; + this.grammarFilePath = grammarFilePath; + + // embeddedLanguages handling + this._embeddedLanguages = Object.create(null); + + if (embeddedLanguages) { + // If embeddedLanguages are configured, fill in `this._embeddedLanguages` + let scopes = Object.keys(embeddedLanguages); + for (let i = 0, len = scopes.length; i < len; i++) { + let scope = scopes[i]; + let language = embeddedLanguages[scope]; + if (typeof language !== 'string') { + // never hurts to be too careful + continue; + } + this._embeddedLanguages[scope] = language; } + } + + // create the regex + let escapedScopes = Object.keys(this._embeddedLanguages).map((scopeName) => strings.escapeRegExpCharacters(scopeName)); + if (escapedScopes.length === 0) { + // no scopes registered + this._embeddedLanguagesRegex = null; + } else { escapedScopes.sort(); escapedScopes.reverse(); - this._cachedScopesRegex = new RegExp(`^((${escapedScopes.join(')|(')}))($|\\.)`, ''); + this._embeddedLanguagesRegex = new RegExp(`^((${escapedScopes.join(')|(')}))($|\\.)`, ''); } - return this._cachedScopesRegex; } /** @@ -126,23 +158,22 @@ export class TMScopeRegistry { if (!scope) { return null; } - let regex = this._getScopesRegex(); - if (!regex) { + if (!this._embeddedLanguagesRegex) { // no scopes registered return null; } - let m = scope.match(regex); + let m = scope.match(this._embeddedLanguagesRegex); if (!m) { // no scopes matched return null; } - let language = this._scopeNameToLanguage[m[1]] || null; - if (language && !this._encounteredLanguages[language]) { - this._encounteredLanguages[language] = true; - this._onDidEncounterLanguage.fire(language); + let language = this._embeddedLanguages[m[1]] || null; + if (!language) { + return null; } + this._registry.onEncounteredLanguage(language); return language; } } @@ -210,11 +241,7 @@ export class MainProcessTextMateSyntax { collector.warn(nls.localize('invalid.path.1', "Expected `contributes.{0}.path` ({1}) to be included inside extension's folder ({2}). This might make the extension non-portable.", grammarsExtPoint.name, normalizedAbsolutePath, extensionFolderPath)); } - this._scopeRegistry.register(syntax.language, syntax.scopeName, normalizedAbsolutePath); - - if (syntax.embeddedLanguages) { - this._scopeRegistry.registerEmbeddedLanguages(syntax.embeddedLanguages); - } + this._scopeRegistry.register(syntax.scopeName, normalizedAbsolutePath, syntax.embeddedLanguages); if (syntax.injectTo) { for (let injectScope of syntax.injectTo) { @@ -245,13 +272,14 @@ export class MainProcessTextMateSyntax { return; } - TokenizationRegistry.register(modeId, createTokenizationSupport(this._scopeRegistry, scopeName, modeId, grammar)); + let languageRegistration = this._scopeRegistry.getLanguageRegistration(scopeName); + TokenizationRegistry.register(modeId, createTokenizationSupport(languageRegistration, modeId, grammar)); }); } } -function createTokenizationSupport(scopeRegistry: TMScopeRegistry, topLevelScopeName: string, modeId: string, grammar: IGrammar): ITokenizationSupport { - var tokenizer = new Tokenizer(scopeRegistry, topLevelScopeName, modeId, grammar); +function createTokenizationSupport(languageRegistration: TMLanguageRegistration, modeId: string, grammar: IGrammar): ITokenizationSupport { + var tokenizer = new Tokenizer(languageRegistration, modeId, grammar); return { getInitialState: () => new TMState(modeId, null, null), tokenize: (line, state, offsetDelta?, stopAtOffset?) => tokenizer.tokenize(line, state, offsetDelta, stopAtOffset) @@ -343,21 +371,21 @@ export class DecodeMap { _decodeMapBrand: void; private lastAssignedTokenId: number; - private scopeRegistry: TMScopeRegistry; + private readonly languageRegistration: TMLanguageRegistration; private readonly scopeToTokenIds: { [scope: string]: TMScopeDecodeData; }; private readonly tokenToTokenId: { [token: string]: number; }; private readonly tokenIdToToken: string[]; prevTokenScopes: TMScopesDecodeData[]; public readonly topLevelScope: TMScopesDecodeData; - constructor(scopeRegistry: TMScopeRegistry, topLevelScopeName: string) { + constructor(languageRegistration: TMLanguageRegistration) { this.lastAssignedTokenId = 0; - this.scopeRegistry = scopeRegistry; + this.languageRegistration = languageRegistration; this.scopeToTokenIds = Object.create(null); this.tokenToTokenId = Object.create(null); this.tokenIdToToken = [null]; this.prevTokenScopes = []; - this.topLevelScope = new TMScopesDecodeData(null, new TMScopeDecodeData(topLevelScopeName, this.scopeRegistry.scopeToLanguage(topLevelScopeName), [])); + this.topLevelScope = new TMScopesDecodeData(null, new TMScopeDecodeData(languageRegistration.scopeName, this.languageRegistration.scopeToLanguage(languageRegistration.scopeName), [])); } private _getTokenId(token: string): number { @@ -383,7 +411,7 @@ export class DecodeMap { tokenIds[i] = this._getTokenId(scopePieces[i]); } - result = new TMScopeDecodeData(scope, this.scopeRegistry.scopeToLanguage(scope), tokenIds); + result = new TMScopeDecodeData(scope, this.languageRegistration.scopeToLanguage(scope), tokenIds); this.scopeToTokenIds[scope] = result; return result; } @@ -420,10 +448,10 @@ class Tokenizer { private _modeId: string; private _decodeMap: DecodeMap; - constructor(scopeRegistry: TMScopeRegistry, topLevelScopeName: string, modeId: string, grammar: IGrammar) { + constructor(languageRegistration: TMLanguageRegistration, modeId: string, grammar: IGrammar) { this._modeId = modeId; this._grammar = grammar; - this._decodeMap = new DecodeMap(scopeRegistry, topLevelScopeName); + this._decodeMap = new DecodeMap(languageRegistration); } public tokenize(line: string, state: TMState, offsetDelta: number = 0, stopAtOffset?: number): ILineTokens { diff --git a/src/vs/editor/test/node/textMate/TMSyntax.test.ts b/src/vs/editor/test/node/textMate/TMSyntax.test.ts index c6a0044f4652336a55d6332c1e39fe02616656f9..7abd3b19aafbb58300d5cfe37e3d9c34a91986d9 100644 --- a/src/vs/editor/test/node/textMate/TMSyntax.test.ts +++ b/src/vs/editor/test/node/textMate/TMSyntax.test.ts @@ -5,7 +5,7 @@ 'use strict'; import * as assert from 'assert'; -import { decodeTextMateToken, decodeTextMateTokens, DecodeMap, TMScopeRegistry } from 'vs/editor/node/textMate/TMSyntax'; +import { decodeTextMateToken, decodeTextMateTokens, DecodeMap, TMScopeRegistry, TMLanguageRegistration } from 'vs/editor/node/textMate/TMSyntax'; import { TMState } from 'vs/editor/common/modes/TMState'; suite('TextMate.TMScopeRegistry', () => { @@ -13,19 +13,19 @@ suite('TextMate.TMScopeRegistry', () => { test('getFilePath', () => { let registry = new TMScopeRegistry(); - registry.register('a', 'source.a', './grammar/a.tmLanguage'); + registry.register('source.a', './grammar/a.tmLanguage'); assert.equal(registry.getFilePath('source.a'), './grammar/a.tmLanguage'); assert.equal(registry.getFilePath('a'), null); assert.equal(registry.getFilePath('source.b'), null); assert.equal(registry.getFilePath('b'), null); - registry.register('b', 'source.b', './grammar/b.tmLanguage'); + registry.register('source.b', './grammar/b.tmLanguage'); assert.equal(registry.getFilePath('source.a'), './grammar/a.tmLanguage'); assert.equal(registry.getFilePath('a'), null); assert.equal(registry.getFilePath('source.b'), './grammar/b.tmLanguage'); assert.equal(registry.getFilePath('b'), null); - registry.register('a', 'source.a', './grammar/ax.tmLanguage'); + registry.register('source.a', './grammar/ax.tmLanguage'); assert.equal(registry.getFilePath('source.a'), './grammar/ax.tmLanguage'); assert.equal(registry.getFilePath('a'), null); assert.equal(registry.getFilePath('source.b'), './grammar/b.tmLanguage'); @@ -34,10 +34,7 @@ suite('TextMate.TMScopeRegistry', () => { test('scopeToLanguage', () => { let registry = new TMScopeRegistry(); - - assert.equal(registry.scopeToLanguage('source.html'), null); - - registry.registerEmbeddedLanguages({ + registry.register('source.html', './grammar/html.tmLanguage', { 'source.html': 'html', 'source.c': 'c', 'source.css': 'css', @@ -46,29 +43,30 @@ suite('TextMate.TMScopeRegistry', () => { 'source.smarty': 'smarty', 'source.baz': null, }); + let languageRegistration = registry.getLanguageRegistration('source.html'); // exact matches - assert.equal(registry.scopeToLanguage('source.html'), 'html'); - assert.equal(registry.scopeToLanguage('source.css'), 'css'); - assert.equal(registry.scopeToLanguage('source.c'), 'c'); - assert.equal(registry.scopeToLanguage('source.js'), 'javascript'); - assert.equal(registry.scopeToLanguage('source.python'), 'python'); - assert.equal(registry.scopeToLanguage('source.smarty'), 'smarty'); + assert.equal(languageRegistration.scopeToLanguage('source.html'), 'html'); + assert.equal(languageRegistration.scopeToLanguage('source.css'), 'css'); + assert.equal(languageRegistration.scopeToLanguage('source.c'), 'c'); + assert.equal(languageRegistration.scopeToLanguage('source.js'), 'javascript'); + assert.equal(languageRegistration.scopeToLanguage('source.python'), 'python'); + assert.equal(languageRegistration.scopeToLanguage('source.smarty'), 'smarty'); // prefix matches - assert.equal(registry.scopeToLanguage('source.css.embedded.html'), 'css'); - assert.equal(registry.scopeToLanguage('source.js.embedded.html'), 'javascript'); - assert.equal(registry.scopeToLanguage('source.python.embedded.html'), 'python'); - assert.equal(registry.scopeToLanguage('source.smarty.embedded.html'), 'smarty'); + assert.equal(languageRegistration.scopeToLanguage('source.css.embedded.html'), 'css'); + assert.equal(languageRegistration.scopeToLanguage('source.js.embedded.html'), 'javascript'); + assert.equal(languageRegistration.scopeToLanguage('source.python.embedded.html'), 'python'); + assert.equal(languageRegistration.scopeToLanguage('source.smarty.embedded.html'), 'smarty'); // misses - assert.equal(registry.scopeToLanguage('source.ts'), null); - assert.equal(registry.scopeToLanguage('source.csss'), null); - assert.equal(registry.scopeToLanguage('source.baz'), null); - assert.equal(registry.scopeToLanguage('asource.css'), null); - assert.equal(registry.scopeToLanguage('a.source.css'), null); - assert.equal(registry.scopeToLanguage('source_css'), null); - assert.equal(registry.scopeToLanguage('punctuation.definition.tag.html'), null); + assert.equal(languageRegistration.scopeToLanguage('source.ts'), null); + assert.equal(languageRegistration.scopeToLanguage('source.csss'), null); + assert.equal(languageRegistration.scopeToLanguage('source.baz'), null); + assert.equal(languageRegistration.scopeToLanguage('asource.css'), null); + assert.equal(languageRegistration.scopeToLanguage('a.source.css'), null); + assert.equal(languageRegistration.scopeToLanguage('source_css'), null); + assert.equal(languageRegistration.scopeToLanguage('punctuation.definition.tag.html'), null); }); }); @@ -77,8 +75,7 @@ suite('TextMate.decodeTextMateTokens', () => { test('embedded modes', () => { let registry = new TMScopeRegistry(); - - registry.registerEmbeddedLanguages({ + registry.register('source.html', './grammar/html.tmLanguage', { 'source.html': 'html', 'source.c': 'c', 'source.css': 'css', @@ -87,8 +84,9 @@ suite('TextMate.decodeTextMateTokens', () => { 'source.smarty': 'smarty', 'source.baz': null, }); + let languageRegistration = registry.getLanguageRegistration('source.html'); - let decodeMap = new DecodeMap(registry, 'source.html'); + let decodeMap = new DecodeMap(languageRegistration); let actual = decodeTextMateTokens( 'texttext', 0, @@ -377,7 +375,7 @@ suite('TextMate.decodeTextMateTokens', () => { let registry = new TMScopeRegistry(); - registry.registerEmbeddedLanguages({ + registry.register('text.html.php', null, { 'text.html': 'html', 'source.php': 'php', 'source.sql': 'sql', @@ -387,7 +385,7 @@ suite('TextMate.decodeTextMateTokens', () => { 'source.css': 'css' }); - let decodeMap = new DecodeMap(registry, 'text.html.php'); + let decodeMap = new DecodeMap(registry.getLanguageRegistration('text.html.php')); for (let i = 0, len = tests.length; i < len; i++) { let test = tests[i]; @@ -818,7 +816,7 @@ suite('TextMate.decodeTextMateTokens', () => { let registry = new TMScopeRegistry(); - registry.registerEmbeddedLanguages({ + registry.register('text.html.basic', null, { 'text.html.basic': 'html', 'source.css': 'css', 'source.js': 'javascript', @@ -826,7 +824,7 @@ suite('TextMate.decodeTextMateTokens', () => { 'source.smarty': 'smarty' }); - let decodeMap = new DecodeMap(registry, 'text.html.basic'); + let decodeMap = new DecodeMap(registry.getLanguageRegistration('text.html.basic')); for (let i = 0, len = tests.length; i < len; i++) { let test = tests[i]; @@ -839,6 +837,75 @@ suite('TextMate.decodeTextMateTokens', () => { assert.deepEqual(actualModeTransitions, test.modeTransitions, 'test ' + test.line); } }); + + test('issue #14661: Comment shortcut in SCSS now using CSS style comments', () => { + let tests = [ + { + line: 'class {', + tmTokens: [ + { startIndex: 0, endIndex: 6, scopes: [ 'source.css.scss' ] }, + { startIndex: 6, endIndex: 7, scopes: [ 'source.css.scss', 'meta.property-list.scss', 'punctuation.section.property-list.begin.bracket.curly.scss' ] } + ], + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 6, type: 'meta.property-list.scss.punctuation.section.begin.bracket.curly' } + ], + modeTransitions: [ + { startIndex: 0, modeId: 'scss' } + ] + }, { + line: ' background: red;', + tmTokens: [ + { startIndex: 0, endIndex: 4, scopes: [ 'source.css.scss', 'meta.property-list.scss' ] }, + { startIndex: 4, endIndex: 14, scopes: [ 'source.css.scss', 'meta.property-list.scss', 'meta.property-name.scss', 'support.type.property-name.scss' ] }, + { startIndex: 14, endIndex: 15, scopes: [ 'source.css.scss', 'meta.property-list.scss', 'punctuation.separator.key-value.scss' ] }, + { startIndex: 15, endIndex: 16, scopes: [ 'source.css.scss', 'meta.property-list.scss' ] }, + { startIndex: 16, endIndex: 19, scopes: [ 'source.css.scss', 'meta.property-list.scss', 'meta.property-value.scss', 'support.constant.color.w3c-standard-color-name.scss' ] }, + { startIndex: 19, endIndex: 20, scopes: [ 'source.css.scss', 'meta.property-list.scss', 'punctuation.terminator.rule.scss' ] } + ], + tokens: [ + { startIndex: 0, type: 'meta.property-list.scss' }, + { startIndex: 4, type: 'meta.property-list.scss.property-name.support.type' }, + { startIndex: 14, type: 'meta.property-list.scss.punctuation.separator.key-value' }, + { startIndex: 15, type: 'meta.property-list.scss' }, + { startIndex: 16, type: 'meta.property-list.scss.support.property-value.constant.color.w3c-standard-color-name' }, + { startIndex: 19, type: 'meta.property-list.scss.punctuation.terminator.rule' } + ], + modeTransitions: [ + { startIndex: 0, modeId: 'scss' } + ] + }, { + line: '}', + tmTokens: [ + { startIndex: 0, endIndex: 1, scopes: [ 'source.css.scss', 'meta.property-list.scss', 'punctuation.section.property-list.end.bracket.curly.scss' ] } + ], + tokens: [ + { startIndex: 0, type: 'meta.property-list.scss.punctuation.section.bracket.curly.end' } + ], + modeTransitions: [ + { startIndex: 0, modeId: 'scss' } + ] + } + ]; + + let registry = new TMScopeRegistry(); + + registry.register('source.css.scss', './syntaxes/scss.json'); + + let decodeMap = new DecodeMap(registry.getLanguageRegistration('source.css.scss')); + + for (let i = 0, len = tests.length; i < len; i++) { + let test = tests[i]; + let actual = decodeTextMateTokens(test.line, 0, decodeMap, test.tmTokens, new TMState('scss', null, null)); + + let actualTokens = actual.tokens.map((t) => { return { startIndex: t.startIndex, type: t.type }; }); + let actualModeTransitions = actual.modeTransitions.map((t) => { return { startIndex: t.startIndex, modeId: t.modeId }; }); + + assert.deepEqual(actualTokens, test.tokens, 'test ' + test.line); + assert.deepEqual(actualModeTransitions, test.modeTransitions, 'test ' + test.line); + } + + }); }); suite('textMate', () => { @@ -874,7 +941,7 @@ suite('textMate', () => { } function testDecodeTextMateToken(input: string[][], expected: string[]): void { - let decodeMap = new DecodeMap(new TMScopeRegistry(), null); + let decodeMap = new DecodeMap(new TMLanguageRegistration(null, null, null, null)); for (let i = 0; i < input.length; i++) { testOneDecodeTextMateToken(decodeMap, input[i], expected[i]);