Make ITheme.getTokenStyleMetadata return an object to be able to express "inherit" (#89600)

src/vs/editor/common/services/modelServiceImpl.ts

@@ -14,7 +14,7 @@ import { Range } from 'vs/editor/common/core/range';
 import { DefaultEndOfLine, EndOfLinePreference, EndOfLineSequence, IIdentifiedSingleEditOperation, ITextBuffer, ITextBufferFactory, ITextModel, ITextModelCreationOptions } from 'vs/editor/common/model';
 import { TextModel, createTextBuffer } from 'vs/editor/common/model/textModel';
 import { IModelLanguageChangedEvent, IModelContentChangedEvent } from 'vs/editor/common/model/textModelEvents';
-import { LanguageIdentifier, DocumentSemanticTokensProviderRegistry, DocumentSemanticTokensProvider, SemanticTokensLegend, SemanticTokens, SemanticTokensEdits, TokenMetadata } from 'vs/editor/common/modes';
+import { LanguageIdentifier, DocumentSemanticTokensProviderRegistry, DocumentSemanticTokensProvider, SemanticTokensLegend, SemanticTokens, SemanticTokensEdits, TokenMetadata, FontStyle, MetadataConsts } from 'vs/editor/common/modes';
 import { PLAINTEXT_LANGUAGE_IDENTIFIER } from 'vs/editor/common/modes/modesRegistry';
 import { ILanguageSelection } from 'vs/editor/common/services/modeService';
 import { IModelService } from 'vs/editor/common/services/modelService';
@@ -629,7 +629,7 @@ class SemanticColoringProviderStyling {
 
 	public getMetadata(tokenTypeIndex: number, tokenModifierSet: number): number {
 		const entry = this._hashTable.get(tokenTypeIndex, tokenModifierSet);
-		let metadata: number | undefined;
+		let metadata: number;
 		if (entry) {
 			metadata = entry.metadata;
 		} else {
@@ -643,9 +643,21 @@ class SemanticColoringProviderStyling {
 				modifierSet = modifierSet >> 1;
 			}
 
-			metadata = this._themeService.getTheme().getTokenStyleMetadata(tokenType, tokenModifiers);
-			if (typeof metadata === 'undefined') {
+			const tokenStyle = this._themeService.getTheme().getTokenStyleMetadata(tokenType, tokenModifiers);
+			if (typeof tokenStyle === 'undefined') {
 				metadata = Constants.NO_STYLING;
+			} else {
+				const fontStyle = (
+					(tokenStyle.italic ? FontStyle.Italic : 0)
+					| (tokenStyle.bold ? FontStyle.Bold : 0)
+					| (tokenStyle.underline ? FontStyle.Underline : 0)
+				);
+				const foreground = tokenStyle.foreground || 0;
+
+				metadata = (
+					foreground << MetadataConsts.FOREGROUND_OFFSET
+					| fontStyle << MetadataConsts.FONT_STYLE_OFFSET
+				) >>> 0;
 			}
 			this._hashTable.add(tokenTypeIndex, tokenModifierSet, metadata);
 		}

src/vs/editor/standalone/browser/standaloneThemeServiceImpl.ts

@@ -13,7 +13,7 @@ import { hc_black, vs, vs_dark } from 'vs/editor/standalone/common/themes';
 import { IEnvironmentService } from 'vs/platform/environment/common/environment';
 import { Registry } from 'vs/platform/registry/common/platform';
 import { ColorIdentifier, Extensions, IColorRegistry } from 'vs/platform/theme/common/colorRegistry';
-import { Extensions as ThemingExtensions, ICssStyleCollector, IIconTheme, IThemingRegistry } from 'vs/platform/theme/common/themeService';
+import { Extensions as ThemingExtensions, ICssStyleCollector, IIconTheme, IThemingRegistry, ITokenStyle } from 'vs/platform/theme/common/themeService';
 import { IDisposable, Disposable } from 'vs/base/common/lifecycle';
 
 const VS_THEME_NAME = 'vs';
@@ -131,7 +131,7 @@ class StandaloneTheme implements IStandaloneTheme {
 		return this._tokenTheme;
 	}
 
-	public getTokenStyleMetadata(type: string, modifiers: string[]): number | undefined {
+	public getTokenStyleMetadata(type: string, modifiers: string[]): ITokenStyle | undefined {
 		return undefined;
 	}
 

src/vs/editor/standalone/test/browser/standaloneLanguages.test.ts

@@ -12,7 +12,7 @@ import { TokenTheme } from 'vs/editor/common/modes/supports/tokenization';
 import { ILineTokens, IToken, TokenizationSupport2Adapter, TokensProvider } from 'vs/editor/standalone/browser/standaloneLanguages';
 import { IStandaloneTheme, IStandaloneThemeData, IStandaloneThemeService } from 'vs/editor/standalone/common/standaloneThemeService';
 import { ColorIdentifier } from 'vs/platform/theme/common/colorRegistry';
-import { IIconTheme, ITheme, LIGHT } from 'vs/platform/theme/common/themeService';
+import { IIconTheme, ITheme, LIGHT, ITokenStyle } from 'vs/platform/theme/common/themeService';
 
 suite('TokenizationSupport2Adapter', () => {
 
@@ -56,7 +56,7 @@ suite('TokenizationSupport2Adapter', () => {
 					throw new Error('Not implemented');
 				},
 
-				getTokenStyleMetadata: (type: string, modifiers: string[]): number | undefined => {
+				getTokenStyleMetadata: (type: string, modifiers: string[]): ITokenStyle | undefined => {
 					return undefined;
 				},
 

src/vs/platform/theme/common/themeService.ts

@@ -79,6 +79,13 @@ export function getThemeTypeSelector(type: ThemeType): string {
 	}
 }
 
+export interface ITokenStyle {
+	readonly foreground?: number;
+	readonly bold?: boolean;
+	readonly underline?: boolean;
+	readonly italic?: boolean;
+}
+
 export interface ITheme {
 	readonly type: ThemeType;
 
@@ -99,7 +106,7 @@ export interface ITheme {
 	/**
 	 * Returns the token style for a given classification. The result uses the <code>MetadataConsts</code> format
 	 */
-	getTokenStyleMetadata(type: string, modifiers: string[]): number | undefined;
+	getTokenStyleMetadata(type: string, modifiers: string[]): ITokenStyle | undefined;
 
 	/**
 	 * List of all colors used with tokens. <code>getTokenStyleMetadata</code> references the colors by index into this list.

src/vs/platform/theme/test/common/testThemeService.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { Event, Emitter } from 'vs/base/common/event';
-import { IThemeService, ITheme, DARK, IIconTheme } from 'vs/platform/theme/common/themeService';
+import { IThemeService, ITheme, DARK, IIconTheme, ITokenStyle } from 'vs/platform/theme/common/themeService';
 import { Color } from 'vs/base/common/color';
 
 export class TestTheme implements ITheme {
@@ -24,7 +24,7 @@ export class TestTheme implements ITheme {
 		throw new Error('Method not implemented.');
 	}
 
-	getTokenStyleMetadata(type: string, modifiers: string[]): number | undefined {
+	getTokenStyleMetadata(type: string, modifiers: string[]): ITokenStyle | undefined {
 		return undefined;
 	}
 

src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts

@@ -17,7 +17,7 @@ import { Position } from 'vs/editor/common/core/position';
 import { Range } from 'vs/editor/common/core/range';
 import { IEditorContribution } from 'vs/editor/common/editorCommon';
 import { ITextModel } from 'vs/editor/common/model';
-import { FontStyle, LanguageIdentifier, StandardTokenType, TokenMetadata, DocumentSemanticTokensProviderRegistry, SemanticTokensLegend, SemanticTokens } from 'vs/editor/common/modes';
+import { FontStyle, LanguageIdentifier, StandardTokenType, TokenMetadata, DocumentSemanticTokensProviderRegistry, SemanticTokensLegend, SemanticTokens, LanguageId } from 'vs/editor/common/modes';
 import { IModeService } from 'vs/editor/common/services/modeService';
 import { INotificationService } from 'vs/platform/notification/common/notification';
 import { editorHoverBackground, editorHoverBorder } from 'vs/platform/theme/common/colorRegistry';
@@ -474,9 +474,30 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget {
 				const modifiers = semanticTokens.legend.tokenModifiers.filter((_, k) => modSet & 1 << k);
 				const range = new Range(line + 1, character + 1, line + 1, character + 1 + len);
 				const definitions = {};
+				const colorMap = this._themeService.getColorTheme().tokenColorMap;
 				const theme = this._themeService.getTheme() as ColorThemeData;
-				const m = theme.getTokenStyleMetadata(type, modifiers, true, definitions);
-				const metadata = this._decodeMetadata(m || 0);
+				const tokenStyle = theme.getTokenStyleMetadata(type, modifiers, true, definitions);
+
+				let fontStyle = FontStyle.None;
+				let foreground: string | undefined = undefined;
+				if (tokenStyle) {
+					fontStyle = (
+						(tokenStyle.italic ? FontStyle.Italic : 0)
+						| (tokenStyle.bold ? FontStyle.Bold : 0)
+						| (tokenStyle.underline ? FontStyle.Underline : 0)
+					);
+					if (tokenStyle.foreground) {
+						foreground = colorMap[tokenStyle.foreground];
+					}
+				}
+
+				const metadata: IDecodedMetadata = {
+					languageIdentifier: this._modeService.getLanguageIdentifier(LanguageId.Null)!,
+					tokenType: StandardTokenType.Other,
+					fontStyle: this._fontStyleToString(fontStyle),
+					foreground: foreground,
+				};
+
 				return { type, modifiers, range, metadata, definitions };
 			}
 			lastLine = line;

src/vs/workbench/services/themes/common/colorThemeData.ts

@@ -13,7 +13,7 @@ import * as types from 'vs/base/common/types';
 import * as objects from 'vs/base/common/objects';
 import * as resources from 'vs/base/common/resources';
 import { Extensions as ColorRegistryExtensions, IColorRegistry, ColorIdentifier, editorBackground, editorForeground } from 'vs/platform/theme/common/colorRegistry';
-import { ThemeType } from 'vs/platform/theme/common/themeService';
+import { ThemeType, ITokenStyle } from 'vs/platform/theme/common/themeService';
 import { Registry } from 'vs/platform/registry/common/platform';
 import { getParseErrorMessage } from 'vs/base/common/jsonErrorMessages';
 import { URI } from 'vs/base/common/uri';
@@ -22,7 +22,6 @@ import { startsWith } from 'vs/base/common/strings';
 import { TokenStyle, TokenClassification, ProbeScope, TokenStylingRule, getTokenClassificationRegistry, TokenStyleValue, TokenStyleData } from 'vs/platform/theme/common/tokenClassificationRegistry';
 import { MatcherWithPriority, Matcher, createMatchers } from 'vs/workbench/services/themes/common/textMateScopeMatcher';
 import { IExtensionResourceLoaderService } from 'vs/workbench/services/extensionResourceLoader/common/extensionResourceLoader';
-import { FontStyle, ColorId, MetadataConsts } from 'vs/editor/common/modes';
 import { CharCode } from 'vs/base/common/charCode';
 
 let colorRegistry = Registry.as<IColorRegistry>(ColorRegistryExtensions.ColorContribution);
@@ -243,27 +242,22 @@ export class ColorThemeData implements IColorTheme {
 		return this.getTokenColorIndex().asArray();
 	}
 
-	public getTokenStyleMetadata(type: string, modifiers: string[], useDefault = true, definitions: TokenStyleDefinitions = {}): number | undefined {
+	public getTokenStyleMetadata(type: string, modifiers: string[], useDefault = true, definitions: TokenStyleDefinitions = {}): ITokenStyle | undefined {
 		const classification = tokenClassificationRegistry.getTokenClassification(type, modifiers);
 		if (!classification) {
 			return undefined;
 		}
 		const style = this.getTokenStyle(classification, useDefault, definitions);
-		let fontStyle = FontStyle.None;
-		let foreground = 0;
-		if (style) {
-			if (style.bold) {
-				fontStyle |= FontStyle.Bold;
-			}
-			if (style.underline) {
-				fontStyle |= FontStyle.Underline;
-			}
-			if (style.italic) {
-				fontStyle |= FontStyle.Italic;
-			}
-			foreground = this.getTokenColorIndex().get(style.foreground);
+		if (!style) {
+			return undefined;
 		}
-		return toMetadata(fontStyle, foreground, 0);
+
+		return {
+			foreground: this.getTokenColorIndex().get(style.foreground),
+			bold: style.bold,
+			underline: style.underline,
+			italic: style.italic
+		};
 	}
 
 	public getTokenStylingRuleScope(rule: TokenStylingRule): 'setting' | 'theme' | undefined {
@@ -819,19 +813,3 @@ function hexUpper(charCode: CharCode): number {
 	}
 	return 0;
 }
-
-function toMetadata(fontStyle: FontStyle, foreground: ColorId | number, background: ColorId | number) {
-	const fontStyleBits = fontStyle << MetadataConsts.FONT_STYLE_OFFSET;
-	const foregroundBits = foreground << MetadataConsts.FOREGROUND_OFFSET;
-	const backgroundBits = background << MetadataConsts.BACKGROUND_OFFSET;
-	if ((fontStyleBits & MetadataConsts.FONT_STYLE_MASK) !== fontStyleBits) {
-		console.log(`Can not express fontStyle ${fontStyle} in metadata`);
-	}
-	if ((backgroundBits & MetadataConsts.BACKGROUND_MASK) !== backgroundBits) {
-		console.log(`Can not express background ${background} in metadata`);
-	}
-	if ((foregroundBits & MetadataConsts.FOREGROUND_MASK) !== foregroundBits) {
-		console.log(`Can not express foreground ${foreground} in metadata`);
-	}
-	return (fontStyleBits | foregroundBits | backgroundBits) >>> 0;
-}

src/vs/workbench/services/themes/test/electron-browser/tokenStyleResolving.test.ts

@@ -16,7 +16,7 @@ import { Schemas } from 'vs/base/common/network';
 import { URI } from 'vs/base/common/uri';
 import { getPathFromAmdModule } from 'vs/base/common/amd';
 import { ExtensionResourceLoaderService } from 'vs/workbench/services/extensionResourceLoader/electron-browser/extensionResourceLoaderService';
-import { TokenMetadata, FontStyle } from 'vs/editor/common/modes';
+import { ITokenStyle } from 'vs/platform/theme/common/themeService';
 
 let tokenClassificationRegistry = getTokenClassificationRegistry();
 
@@ -49,24 +49,21 @@ function assertTokenStyle(actual: TokenStyle | undefined | null, expected: Token
 	assert.equal(tokenStyleAsString(actual), tokenStyleAsString(expected), message);
 }
 
-function assertTokenStyleMetaData(colorIndex: string[], actual: number | undefined, expected: TokenStyle | undefined | null, message?: string) {
+function assertTokenStyleMetaData(colorIndex: string[], actual: ITokenStyle | undefined, expected: TokenStyle | undefined | null, message?: string) {
 	if (expected === undefined || expected === null || actual === undefined) {
 		assert.equal(actual, expected, message);
 		return;
 	}
-	const actualFontStyle = TokenMetadata.getFontStyle(actual);
-	assert.equal((actualFontStyle & FontStyle.Bold) === FontStyle.Bold, expected.bold === true, 'bold');
-	assert.equal((actualFontStyle & FontStyle.Italic) === FontStyle.Italic, expected.italic === true, 'italic');
-	assert.equal((actualFontStyle & FontStyle.Underline) === FontStyle.Underline, expected.underline === true, 'underline');
+	assert.strictEqual(actual.bold, expected.bold, 'bold');
+	assert.strictEqual(actual.italic, expected.italic, 'italic');
+	assert.strictEqual(actual.underline, expected.underline, 'underline');
 
-	const actualForegroundIndex = TokenMetadata.getForeground(actual);
+	const actualForegroundIndex = actual.foreground;
 	if (expected.foreground) {
 		assert.equal(actualForegroundIndex, colorIndex.indexOf(Color.Format.CSS.formatHexA(expected.foreground, true).toUpperCase()), 'foreground');
 	} else {
 		assert.equal(actualForegroundIndex, 0, 'foreground');
 	}
-	const actualBackgroundIndex = TokenMetadata.getBackground(actual);
-	assert.equal(actualBackgroundIndex, 0, 'background');
 }