connect ModelSemanticColoring to Theme

src/vs/editor/common/services/modelServiceImpl.ts

@@ -14,7 +14,7 @@ import { Range } from 'vs/editor/common/core/range';
 import { DefaultEndOfLine, EndOfLinePreference, EndOfLineSequence, IIdentifiedSingleEditOperation, ITextBuffer, ITextBufferFactory, ITextModel, ITextModelCreationOptions } from 'vs/editor/common/model';
 import { TextModel, createTextBuffer } from 'vs/editor/common/model/textModel';
 import { IModelLanguageChangedEvent, IModelContentChangedEvent } from 'vs/editor/common/model/textModelEvents';
-import { LanguageIdentifier, SemanticColoringProviderRegistry, SemanticColoringProvider, SemanticColoring, FontStyle, MetadataConsts } from 'vs/editor/common/modes';
+import { LanguageIdentifier, SemanticColoringProviderRegistry, SemanticColoringProvider, SemanticColoring, SemanticColoringLegend } from 'vs/editor/common/modes';
 import { PLAINTEXT_LANGUAGE_IDENTIFIER } from 'vs/editor/common/modes/modesRegistry';
 import { ILanguageSelection } from 'vs/editor/common/services/modeService';
 import { IModelService } from 'vs/editor/common/services/modelService';
@@ -23,6 +23,7 @@ import { IConfigurationService } from 'vs/platform/configuration/common/configur
 import { RunOnceScheduler } from 'vs/base/common/async';
 import { CancellationTokenSource } from 'vs/base/common/cancellation';
 import { SparseEncodedTokens, MultilineTokens2 } from 'vs/editor/common/model/tokensStore';
+import { IThemeService } from 'vs/platform/theme/common/themeService';
 
 function MODEL_ID(resource: URI): string {
 	return resource.toString();
@@ -118,7 +119,8 @@ export class ModelServiceImpl extends Disposable implements IModelService {
 
 	constructor(
 		@IConfigurationService configurationService: IConfigurationService,
-		@ITextResourcePropertiesService resourcePropertiesService: ITextResourcePropertiesService
+		@ITextResourcePropertiesService resourcePropertiesService: ITextResourcePropertiesService,
+		@IThemeService themeService: IThemeService
 	) {
 		super();
 		this._configurationService = configurationService;
@@ -129,7 +131,7 @@ export class ModelServiceImpl extends Disposable implements IModelService {
 		this._configurationServiceSubscription = this._configurationService.onDidChangeConfiguration(e => this._updateModelOptions());
 		this._updateModelOptions();
 
-		this._register(new SemanticColoringFeature(this));
+		this._register(new SemanticColoringFeature(this, themeService));
 	}
 
 	private static _readModelOptions(config: IRawConfig, isForSimpleWidget: boolean): ITextModelCreationOptions {
@@ -440,11 +442,11 @@ export interface ILineSequence {
 class SemanticColoringFeature extends Disposable {
 	private _watchers: Record<string, ModelSemanticColoring>;
 
-	constructor(modelService: IModelService) {
+	constructor(modelService: IModelService, themeService: IThemeService) {
 		super();
 		this._watchers = Object.create(null);
 		this._register(modelService.onModelAdded((model) => {
-			this._watchers[model.uri.toString()] = new ModelSemanticColoring(model);
+			this._watchers[model.uri.toString()] = new ModelSemanticColoring(model, themeService);
 		}));
 		this._register(modelService.onModelRemoved((model) => {
 			this._watchers[model.uri.toString()].dispose();
@@ -460,8 +462,9 @@ class ModelSemanticColoring extends Disposable {
 	private readonly _fetchSemanticTokens: RunOnceScheduler;
 	private _currentResponse: SemanticColoring | null;
 	private _currentRequestCancellationTokenSource: CancellationTokenSource | null;
+	private _themeService: IThemeService;
 
-	constructor(model: ITextModel) {
+	constructor(model: ITextModel, themeService: IThemeService) {
 		super();
 
 		this._isDisposed = false;
@@ -469,6 +472,7 @@ class ModelSemanticColoring extends Disposable {
 		this._fetchSemanticTokens = this._register(new RunOnceScheduler(() => this._fetchSemanticTokensNow(), 500));
 		this._currentResponse = null;
 		this._currentRequestCancellationTokenSource = null;
+		this._themeService = themeService;
 
 		this._register(this._model.onDidChangeContent(e => this._fetchSemanticTokens.schedule()));
 		this._register(SemanticColoringProviderRegistry.onDidChange(e => this._fetchSemanticTokens.schedule()));
@@ -509,16 +513,16 @@ class ModelSemanticColoring extends Disposable {
 		request.then((res) => {
 			this._currentRequestCancellationTokenSource = null;
 			contentChangeListener.dispose();
-			this._setSemanticTokens(res || null, pendingChanges);
+			this._setSemanticTokens(res || null, provider.getLegend(), pendingChanges);
 		}, (err) => {
 			errors.onUnexpectedError(err);
 			this._currentRequestCancellationTokenSource = null;
 			contentChangeListener.dispose();
-			this._setSemanticTokens(null, pendingChanges);
+			this._setSemanticTokens(null, provider.getLegend(), pendingChanges);
 		});
 	}
 
-	private _setSemanticTokens(tokens: SemanticColoring | null, pendingChanges: IModelContentChangedEvent[]): void {
+	private _setSemanticTokens(tokens: SemanticColoring | null, legend: SemanticColoringLegend, pendingChanges: IModelContentChangedEvent[]): void {
 		if (this._currentResponse) {
 			this._currentResponse.dispose();
 			this._currentResponse = null;
@@ -540,30 +544,37 @@ class ModelSemanticColoring extends Disposable {
 		for (const area of this._currentResponse.areas) {
 			const srcTokens = area.data;
 			const tokenCount = srcTokens.length / 5;
-			const destTokens = new Uint32Array(4 * tokenCount);
+			let destTokens = new Uint32Array(4 * tokenCount);
+			let destOffset = 0;
 			for (let i = 0; i < tokenCount; i++) {
 				const srcOffset = 5 * i;
 				const deltaLine = srcTokens[srcOffset];
 				const startCharacter = srcTokens[srcOffset + 1];
 				const endCharacter = srcTokens[srcOffset + 2];
-				// const tokenType = srcTokens[srcOffset + 3];
-				// const tokenModifiers = srcTokens[srcOffset + 4];
-				// TODO@semantic: map here tokenType and tokenModifiers to metadata
-
-				const fontStyle = FontStyle.Italic | FontStyle.Bold | FontStyle.Underline;
-				const foregroundColorId = 3;
-				const metadata = (
-					(fontStyle << MetadataConsts.FONT_STYLE_OFFSET)
-					| (foregroundColorId << MetadataConsts.FOREGROUND_OFFSET)
-				) >>> 0;
-
-				const destOffset = 4 * i;
-				destTokens[destOffset] = deltaLine;
-				destTokens[destOffset + 1] = startCharacter;
-				destTokens[destOffset + 2] = endCharacter;
-				destTokens[destOffset + 3] = metadata;
+				const tokenTypeIndex = srcTokens[srcOffset + 3];
+				const tokenType = legend.tokenTypes[tokenTypeIndex];
+
+				const tokenModifierSet = srcTokens[srcOffset + 4];
+				let tokenModifiers: string[] = [];
+				for (let modifierIndex = 0; tokenModifierSet !== 0 && modifierIndex < legend.tokenModifiers.length; modifierIndex++) {
+					if (tokenModifierSet & 1) {
+						tokenModifiers.push(legend.tokenTypes[modifierIndex]);
+					}
+				}
+
+				const metadata = this._themeService.getTheme().getTokenStyleMetadata(tokenType, tokenModifiers);
+				if (metadata !== undefined) {
+					destTokens[destOffset] = deltaLine;
+					destTokens[destOffset + 1] = startCharacter;
+					destTokens[destOffset + 2] = endCharacter;
+					destTokens[destOffset + 3] = metadata;
+					destOffset += 4;
+				}
 			}
 
+			if (destOffset !== destTokens.length) {
+				destTokens = destTokens.subarray(0, destOffset);
+			}
 			const tokens = new MultilineTokens2(area.line, new SparseEncodedTokens(destTokens));
 			result.push(tokens);
 		}

src/vs/editor/standalone/browser/standaloneServices.ts

@@ -144,11 +144,11 @@ export module StaticServices {
 
 	export const modeService = define(IModeService, (o) => new ModeServiceImpl());
 
-	export const modelService = define(IModelService, (o) => new ModelServiceImpl(configurationService.get(o), resourcePropertiesService.get(o)));
+	export const standaloneThemeService = define(IStandaloneThemeService, () => new StandaloneThemeServiceImpl());
 
-	export const markerDecorationsService = define(IMarkerDecorationsService, (o) => new MarkerDecorationsService(modelService.get(o), markerService.get(o)));
+	export const modelService = define(IModelService, (o) => new ModelServiceImpl(configurationService.get(o), resourcePropertiesService.get(o), standaloneThemeService.get(o)));
 
-	export const standaloneThemeService = define(IStandaloneThemeService, () => new StandaloneThemeServiceImpl());
+	export const markerDecorationsService = define(IMarkerDecorationsService, (o) => new MarkerDecorationsService(modelService.get(o), markerService.get(o)));
 
 	export const codeEditorService = define(ICodeEditorService, (o) => new StandaloneCodeEditorServiceImpl(standaloneThemeService.get(o)));
 

src/vs/editor/standalone/browser/standaloneThemeServiceImpl.ts

@@ -129,6 +129,14 @@ class StandaloneTheme implements IStandaloneTheme {
 		}
 		return this._tokenTheme;
 	}
+
+	public getTokenStyleMetadata(type: string, modifiers: string[]): number | undefined {
+		return undefined;
+	}
+
+	public get tokenColorMap(): string[] {
+		return [];
+	}
 }
 
 function isBuiltinTheme(themeName: string): themeName is BuiltinTheme {

src/vs/editor/standalone/test/browser/standaloneLanguages.test.ts

@@ -54,7 +54,13 @@ suite('TokenizationSupport2Adapter', () => {
 
 				defines: (color: ColorIdentifier): boolean => {
 					throw new Error('Not implemented');
-				}
+				},
+
+				getTokenStyleMetadata: (type: string, modifiers: string[]): number | undefined => {
+					return undefined;
+				},
+
+				tokenColorMap: []
 			};
 		}
 

src/vs/platform/theme/common/themeService.ts

@@ -59,6 +59,16 @@ export interface ITheme {
 	 * default color will be used.
 	 */
 	defines(color: ColorIdentifier): boolean;
+
+	/**
+	 * Returns the token style for a given classification. The result uses the <code>MetadataConsts</code> format
+	 */
+	getTokenStyleMetadata(type: string, modifiers: string[]): number | undefined;
+
+	/**
+	 * List of all colors used with tokens. <code>getTokenStyleMetadata</code> references the colors by index into this list.
+	 */
+	readonly tokenColorMap: string[];
 }
 
 export interface IIconTheme {

src/vs/platform/theme/test/common/testThemeService.ts

@@ -23,6 +23,14 @@ export class TestTheme implements ITheme {
 	defines(color: string): boolean {
 		throw new Error('Method not implemented.');
 	}
+
+	getTokenStyleMetadata(type: string, modifiers: string[]): number | undefined {
+		return undefined;
+	}
+
+	get tokenColorMap(): string[] {
+		return [];
+	}
 }
 
 export class TestIconTheme implements IIconTheme {

src/vs/workbench/contrib/terminal/test/electron-browser/terminalColorRegistry.test.ts

@@ -20,8 +20,8 @@ function getMockTheme(type: ThemeType): ITheme {
 		type: type,
 		getColor: (colorId: ColorIdentifier): Color | undefined => themingRegistry.resolveDefaultColor(colorId, theme),
 		defines: () => true,
-		getTokenStyle: () => undefined,
-		resolveScopes: () => undefined
+		getTokenStyleMetadata: () => undefined,
+		tokenColorMap: []
 
 	};
 	return theme;

src/vs/workbench/services/themes/common/colorThemeData.ts

@@ -229,7 +229,11 @@ export class ColorThemeData implements IColorTheme {
 		return this.getTokenColorIndex().asArray();
 	}
 
-	public getTokenStyleMetadata(classification: TokenClassification, useDefault?: boolean): number {
+	public getTokenStyleMetadata(type: string, modifiers: string[], useDefault?: boolean): number | undefined {
+		const classification = tokenClassificationRegistry.getTokenClassification(type, modifiers);
+		if (!classification) {
+			return undefined;
+		}
 		const style = this.getTokenStyle(classification, useDefault);
 		let fontStyle = FontStyle.None;
 		let foreground = 0;

src/vs/workbench/services/themes/common/workbenchThemeService.ts

@@ -8,7 +8,6 @@ import { Event } from 'vs/base/common/event';
 import { Color } from 'vs/base/common/color';
 import { ITheme, IThemeService, IIconTheme } from 'vs/platform/theme/common/themeService';
 import { ConfigurationTarget } from 'vs/platform/configuration/common/configuration';
-import { TokenClassification } from 'vs/platform/theme/common/tokenClassificationRegistry';
 
 export const IWorkbenchThemeService = createDecorator<IWorkbenchThemeService>('themeService');
 
@@ -33,16 +32,6 @@ export interface IColorTheme extends ITheme {
 	readonly description?: string;
 	readonly isLoaded: boolean;
 	readonly tokenColors: ITextMateThemingRule[];
-
-	/**
-	 * Returns the token style for a given classification. The result uses the <code>MetadataConsts</code> format
-	 */
-	getTokenStyleMetadata(classification: TokenClassification): number;
-
-	/**
-	 * List of all colors used with tokens. <code>getTokenStyleMetadata</code> references the colors by index into this list.
-	 */
-	readonly tokenColorMap: string[];
 }
 
 export interface IColorMap {

src/vs/workbench/services/themes/test/electron-browser/tokenStyleResolving.test.ts

@@ -49,9 +49,9 @@ function assertTokenStyle(actual: TokenStyle | undefined | null, expected: Token
 	assert.equal(tokenStyleAsString(actual), tokenStyleAsString(expected), message);
 }
 
-function assertTokenStyleMetaData(colorIndex: string[], actual: number, expected: TokenStyle | undefined | null, message?: string) {
-	if (!expected) {
-		assert.equal(actual, 0);
+function assertTokenStyleMetaData(colorIndex: string[], actual: number | undefined, expected: TokenStyle | undefined | null, message?: string) {
+	if (!expected || !actual) {
+		assert.equal(actual, expected);
 		return;
 	}
 	const actualFontStyle = TokenMetadata.getFontStyle(actual);
@@ -74,14 +74,17 @@ function assertTokenStyles(themeData: ColorThemeData, expected: { [qualifiedClas
 	const colorIndex = themeData.tokenColorMap;
 
 	for (let qualifiedClassifier in expected) {
-		const classification = tokenClassificationRegistry.getTokenClassificationFromString(qualifiedClassifier);
+		const modifiers = qualifiedClassifier.split('.');
+		const type = modifiers.shift()!;
+
+		const classification = tokenClassificationRegistry.getTokenClassification(type, modifiers);
 		assert.ok(classification, 'Classification not found');
 
 		const tokenStyle = themeData.getTokenStyle(classification!);
 		const expectedTokenStyle = expected[qualifiedClassifier];
 		assertTokenStyle(tokenStyle, expectedTokenStyle, qualifiedClassifier);
 
-		const tokenStyleMetaData = themeData.getTokenStyleMetadata(classification!);
+		const tokenStyleMetaData = themeData.getTokenStyleMetadata(type, modifiers);
 		assertTokenStyleMetaData(colorIndex, tokenStyleMetaData, expectedTokenStyle);
 	}
 }