Adopt Color in tokenization registry map

src/vs/base/common/color.ts

@@ -32,6 +32,15 @@ export class RGBA {
 		this.a = RGBA._clampInt_0_255(a);
 	}
 
+	public static equals(a: RGBA, b: RGBA): boolean {
+		return (
+			a.r === b.r
+			&& a.g === b.g
+			&& a.b === b.b
+			&& a.a === b.a
+		);
+	}
+
 	private static _clampInt_0_255(c: number): number {
 		if (c < 0) {
 			return 0;
@@ -228,20 +237,6 @@ function _hue2rgb(p: number, q: number, t: number) {
 	return p;
 }
 
-/**
- * @param hex string (#RRGGBB or #RRGGBBAA).
- */
-export function hexToCSSrgba(hex: string) {
-	if (hex.length === 9) {
-		return toCSSrgba(hex2rgba(hex));
-	}
-	return hex;
-}
-
-function toCSSrgba(rgba: RGBA): string {
-	return `rgba(${rgba.r}, ${rgba.g}, ${rgba.b}, ${+(rgba.a / 255).toFixed(2)})`;
-}
-
 export class Color {
 
 	public static fromRGBA(rgba: RGBA): Color {
@@ -271,6 +266,13 @@ export class Color {
 		this.hsla = null;
 	}
 
+	public equals(other: Color): boolean {
+		if (!other) {
+			return false;
+		}
+		return RGBA.equals(this.rgba, other.rgba);
+	}
+
 	/**
 	 * http://www.w3.org/TR/WCAG20/#relativeluminancedef
 	 * Returns the number in the set [0, 1]. O => Darkest Black. 1 => Lightest white.
@@ -355,7 +357,32 @@ export class Color {
 	}
 
 	public toString(): string {
-		return toCSSrgba(this.rgba);
+		const rgba = this.rgba;
+		return `rgba(${rgba.r}, ${rgba.g}, ${rgba.b}, ${+(rgba.a / 255).toFixed(2)})`;
+	}
+
+	/**
+	 * Prins the color as #RRGGBB
+	 */
+	public toRGBHex(): string {
+		const rgba = this.rgba;
+		return `#${Color._toTwoDigitHex(rgba.r)}${Color._toTwoDigitHex(rgba.g)}${Color._toTwoDigitHex(rgba.b)}`;
+	}
+
+	/**
+	 * Prins the color as #RRGGBBAA
+	 */
+	public toRGBAHex(): string {
+		const rgba = this.rgba;
+		return `#${Color._toTwoDigitHex(rgba.r)}${Color._toTwoDigitHex(rgba.g)}${Color._toTwoDigitHex(rgba.b)}${Color._toTwoDigitHex(rgba.a)}`;
+	}
+
+	private static _toTwoDigitHex(n: number): string {
+		let r = n.toString(16);
+		if (r.length !== 2) {
+			return '0' + r;
+		}
+		return r;
 	}
 
 	public toHSLA(): HSLA {

src/vs/editor/browser/services/standaloneColorServiceImpl.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
-import { Theme, IThemeRule } from 'vs/editor/common/modes/supports/tokenization';
+import { Theme, IThemeRule, generateTokensCSSForColorMap } from 'vs/editor/common/modes/supports/tokenization';
 import { IStandaloneColorService, BuiltinTheme, ITheme } from 'vs/editor/common/services/standaloneColorService';
 import { vs, vs_dark, hc_black } from 'vs/editor/common/standalone/themes';
 import * as dom from 'vs/base/browser/dom';
@@ -61,18 +61,6 @@ export class StandaloneColorServiceImpl implements IStandaloneColorService {
 		this.setTheme(VS_THEME_NAME);
 	}
 
-	private static _generateCSS(colorMap: string[]): string {
-		let rules: string[] = [];
-		for (let i = 1, len = colorMap.length; i < len; i++) {
-			let color = colorMap[i];
-			rules[i] = `.mtk${i} { color: #${color}; }`;
-		}
-		rules.push('.mtki { font-style: italic; }');
-		rules.push('.mtkb { font-weight: bold; }');
-		rules.push('.mtku { text-decoration: underline; }');
-		return rules.join('\n');
-	}
-
 	public defineTheme(themeName: string, themeData: ITheme): void {
 		if (!/^[a-z0-9\-]+$/i.test(themeName) || isBuiltinTheme(themeName)) {
 			throw new Error('Illegal theme name!');
@@ -107,7 +95,7 @@ export class StandaloneColorServiceImpl implements IStandaloneColorService {
 
 		this._theme = Theme.createFromRawTheme(themeData.rules);
 		let colorMap = this._theme.getColorMap();
-		let cssRules = StandaloneColorServiceImpl._generateCSS(colorMap);
+		let cssRules = generateTokensCSSForColorMap(colorMap);
 		this._styleElement.innerHTML = cssRules;
 
 		TokenizationRegistry.setColorMap(colorMap);

src/vs/editor/browser/standalone/colorizer.ts

@@ -159,12 +159,11 @@ function _fakeColorize(lines: string[], tabSize: number): string {
 function _actualColorize(lines: string[], tabSize: number, tokenizationSupport: ITokenizationSupport): string {
 	let html: string[] = [];
 	let state = tokenizationSupport.getInitialState();
-	let colorMap = TokenizationRegistry.getColorMap();
 
 	for (let i = 0, length = lines.length; i < length; i++) {
 		let line = lines[i];
 		let tokenizeResult = tokenizationSupport.tokenize2(line, state, 0);
-		let lineTokens = new LineTokens(colorMap, tokenizeResult.tokens, line);
+		let lineTokens = new LineTokens(tokenizeResult.tokens, line);
 		let renderResult = renderViewLine(new RenderLineInput(
 			false,
 			line,

src/vs/editor/common/core/lineTokens.ts

@@ -11,7 +11,6 @@ import { ColorId, FontStyle, StandardTokenType, LanguageId } from 'vs/editor/com
 export class LineToken {
 	_lineTokenBrand: void;
 
-	private readonly _colorMap: string[];
 	private readonly _source: LineTokens;
 	private readonly _tokenIndex: number;
 	private readonly _metadata: number;
@@ -38,20 +37,11 @@ export class LineToken {
 		return TokenMetadata.getForeground(this._metadata);
 	}
 
-	public get foreground(): string {
-		return this._colorMap[this.foregroundId];
-	}
-
 	public get backgroundId(): ColorId {
 		return TokenMetadata.getBackground(this._metadata);
 	}
 
-	public get background(): string {
-		return this._colorMap[this.backgroundId];
-	}
-
-	constructor(colorMap: string[], source: LineTokens, tokenIndex: number, tokenCount: number, startOffset: number, endOffset: number, metadata: number) {
-		this._colorMap = colorMap;
+	constructor(source: LineTokens, tokenIndex: number, tokenCount: number, startOffset: number, endOffset: number, metadata: number) {
 		this._source = source;
 		this._tokenIndex = tokenIndex;
 		this._metadata = metadata;
@@ -83,14 +73,12 @@ export class LineToken {
 export class LineTokens {
 	_lineTokensBrand: void;
 
-	private readonly _colorMap: string[];
 	private readonly _tokens: Uint32Array;
 	private readonly _tokensCount: number;
 	private readonly _text: string;
 	private readonly _textLength: number;
 
-	constructor(colorMap: string[], tokens: Uint32Array, text: string) {
-		this._colorMap = colorMap;
+	constructor(tokens: Uint32Array, text: string) {
 		this._tokens = tokens;
 		this._tokensCount = (this._tokens.length >>> 1);
 		this._text = text;
@@ -159,7 +147,7 @@ export class LineTokens {
 			endOffset = this._textLength;
 		}
 		let metadata = this._tokens[(tokenIndex << 1) + 1];
-		return new LineToken(this._colorMap, this, tokenIndex, this._tokensCount, startOffset, endOffset, metadata);
+		return new LineToken(this, tokenIndex, this._tokensCount, startOffset, endOffset, metadata);
 	}
 
 	public firstToken(): LineToken {

src/vs/editor/common/model/modelLine.ts

@@ -258,16 +258,16 @@ export class ModelLine {
 		this._lineTokens = tokens.buffer;
 	}
 
-	public getTokens(topLevelLanguageId: LanguageId, colorMap: string[]): LineTokens {
+	public getTokens(topLevelLanguageId: LanguageId): LineTokens {
 		let rawLineTokens = this._lineTokens;
 		if (rawLineTokens) {
-			return new LineTokens(colorMap, new Uint32Array(rawLineTokens), this._text);
+			return new LineTokens(new Uint32Array(rawLineTokens), this._text);
 		}
 
 		let lineTokens = new Uint32Array(2);
 		lineTokens[0] = 0;
 		lineTokens[1] = ModelLine._getDefaultMetadata(topLevelLanguageId);
-		return new LineTokens(colorMap, lineTokens, this._text);
+		return new LineTokens(lineTokens, this._text);
 	}
 
 	// --- END TOKENS

src/vs/editor/common/model/textModelWithTokens.ts

@@ -63,7 +63,6 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 
 	private _languageIdentifier: LanguageIdentifier;
 	private _tokenizationListener: IDisposable;
-	private _colorMap: string[];
 	private _tokenizationSupport: ITokenizationSupport;
 
 	private _invalidLineStartIndex: number;
@@ -140,7 +139,6 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 			}
 		}
 
-		this._colorMap = TokenizationRegistry.getColorMap();
 		this._lastState = null;
 		this._invalidLineStartIndex = 0;
 		this._beginBackgroundTokenization();
@@ -183,7 +181,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 	}
 
 	private _getLineTokens(lineNumber: number): LineTokens {
-		return this._lines[lineNumber - 1].getTokens(this._languageIdentifier.id, this._colorMap);
+		return this._lines[lineNumber - 1].getTokens(this._languageIdentifier.id);
 	}
 
 	public getLanguageIdentifier(): LanguageIdentifier {

src/vs/editor/common/modes.ts

@@ -15,6 +15,7 @@ import { Position } from 'vs/editor/common/core/position';
 import { Range } from 'vs/editor/common/core/range';
 import Event from 'vs/base/common/event';
 import { TokenizationRegistryImpl } from 'vs/editor/common/modes/tokenizationRegistry';
+import { Color } from 'vs/base/common/color';
 
 /**
  * Open ended enum at runtime
@@ -838,9 +839,9 @@ export interface ITokenizationRegistry {
 	/**
 	 * Set the new color map that all tokens will use in their ColorId binary encoded bits for foreground and background.
 	 */
-	setColorMap(colorMap: string[]): void;
+	setColorMap(colorMap: Color[]): void;
 
-	getColorMap(): string[];
+	getColorMap(): Color[];
 }
 
 /**

src/vs/editor/common/modes/supports/tokenization.ts

@@ -5,6 +5,7 @@
 'use strict';
 
 import { ColorId, FontStyle, MetadataConsts, LanguageId, StandardTokenType } from 'vs/editor/common/modes';
+import { Color } from 'vs/base/common/color';
 
 export interface IThemeRule {
 	token: string;
@@ -141,7 +142,7 @@ function resolveParsedThemeRules(parsedThemeRules: ParsedThemeRule[]): Theme {
 export class ColorMap {
 
 	private _lastColorId: number;
-	private _id2color: string[];
+	private _id2color: Color[];
 	private _color2id: Map<string, ColorId>;
 
 	constructor() {
@@ -164,11 +165,11 @@ export class ColorMap {
 		}
 		value = ++this._lastColorId;
 		this._color2id.set(color, value);
-		this._id2color[value] = color;
+		this._id2color[value] = Color.fromHex('#' + color);
 		return value;
 	}
 
-	public getColorMap(): string[] {
+	public getColorMap(): Color[] {
 		return this._id2color.slice(0);
 	}
 
@@ -194,7 +195,7 @@ export class Theme {
 		this._cache = new Map<string, number>();
 	}
 
-	public getColorMap(): string[] {
+	public getColorMap(): Color[] {
 		return this._colorMap.getColorMap();
 	}
 
@@ -389,3 +390,15 @@ export class ThemeTrieElement {
 		child.insert(tail, fontStyle, foreground, background);
 	}
 }
+
+export function generateTokensCSSForColorMap(colorMap: Color[]): string {
+	let rules: string[] = [];
+	for (let i = 1, len = colorMap.length; i < len; i++) {
+		let color = colorMap[i];
+		rules[i] = `.mtk${i} { color: ${color.toString()}; }`;
+	}
+	rules.push('.mtki { font-style: italic; }');
+	rules.push('.mtkb { font-weight: bold; }');
+	rules.push('.mtku { text-decoration: underline; }');
+	return rules.join('\n');
+}

src/vs/editor/common/modes/textToHtmlTokenizer.ts

@@ -127,7 +127,7 @@ function _tokenizeToString(text: string, tokenizationSupport: ITokenizationSuppo
 		}
 
 		let tokenizationResult = tokenizationSupport.tokenize2(line, currentState, 0);
-		let lineTokens = new LineTokens(null, tokenizationResult.tokens, line);
+		let lineTokens = new LineTokens(tokenizationResult.tokens, line);
 		let viewLineTokens = lineTokens.inflate();
 
 		let startOffset = 0;

src/vs/editor/common/modes/tokenizationRegistry.ts

@@ -7,6 +7,7 @@
 import { IDisposable } from 'vs/base/common/lifecycle';
 import Event, { Emitter } from 'vs/base/common/event';
 import { ITokenizationRegistry, ITokenizationSupport, ITokenizationSupportChangedEvent } from 'vs/editor/common/modes';
+import { Color } from 'vs/base/common/color';
 
 export class TokenizationRegistryImpl implements ITokenizationRegistry {
 
@@ -15,7 +16,7 @@ export class TokenizationRegistryImpl implements ITokenizationRegistry {
 	private _onDidChange: Emitter<ITokenizationSupportChangedEvent> = new Emitter<ITokenizationSupportChangedEvent>();
 	public onDidChange: Event<ITokenizationSupportChangedEvent> = this._onDidChange.event;
 
-	private _colorMap: string[];
+	private _colorMap: Color[];
 
 	constructor() {
 		this._map = Object.create(null);
@@ -47,15 +48,15 @@ export class TokenizationRegistryImpl implements ITokenizationRegistry {
 		return (this._map[language] || null);
 	}
 
-	public setColorMap(colorMap: string[]): void {
+	public setColorMap(colorMap: Color[]): void {
 		this._colorMap = colorMap;
 		this._onDidChange.fire({
 			changedLanguages: Object.keys(this._map),
-			changedColorMap: false
+			changedColorMap: true
 		});
 	}
 
-	public getColorMap(): string[] {
+	public getColorMap(): Color[] {
 		return this._colorMap;
 	}
 }

src/vs/editor/common/view/minimapCharRenderer.ts

@@ -7,6 +7,7 @@
 import { CharCode } from 'vs/base/common/charCode';
 import { ColorId, TokenizationRegistry } from 'vs/editor/common/modes';
 import Event, { Emitter } from 'vs/base/common/event';
+import { Color } from 'vs/base/common/color';
 
 export class ParsedColor {
 
@@ -44,38 +45,25 @@ export class MinimapTokensColorTracker {
 		return this._INSTANCE;
 	}
 
-	private _lastColorMap: string[];
 	private _colors: ParsedColor[];
 
 	private _onDidChange = new Emitter<void>();
 	public onDidChange: Event<void> = this._onDidChange.event;
 
 	private constructor() {
-		this._lastColorMap = [];
 		this._setColorMap(TokenizationRegistry.getColorMap());
-		TokenizationRegistry.onDidChange(() => this._setColorMap(TokenizationRegistry.getColorMap()));
-	}
-
-	private static _equals(a: string[], b: string[]): boolean {
-		if (a.length !== b.length) {
-			return false;
-		}
-		for (let i = 0, len = a.length; i < len; i++) {
-			if (a[i] !== b[i]) {
-				return false;
+		TokenizationRegistry.onDidChange((e) => {
+			if (e.changedColorMap) {
+				this._setColorMap(TokenizationRegistry.getColorMap());
 			}
-		}
-		return true;
+		});
 	}
 
-	private _setColorMap(colorMap: string[]): void {
-		if (MinimapTokensColorTracker._equals(this._lastColorMap, colorMap)) {
-			return;
-		}
-		this._lastColorMap = colorMap.slice(0);
+	private _setColorMap(colorMap: Color[]): void {
 		this._colors = [null];
 		for (let colorId = 1; colorId < colorMap.length; colorId++) {
-			this._colors[colorId] = MinimapTokensColorTracker._parseColor(colorMap[colorId]);
+			const color = colorMap[colorId].toRGBA();
+			this._colors[colorId] = new ParsedColor(color.r, color.g, color.b);
 		}
 		this._onDidChange.fire(void 0);
 	}

src/vs/editor/common/viewModel/viewModelImpl.ts

@@ -601,17 +601,14 @@ export class ViewModel extends EventEmitter implements IViewModel {
 		let colorMap = TokenizationRegistry.getColorMap();
 		for (let i = 1, len = colorMap.length; i < len; i++) {
 			let color = colorMap[i];
-			if (/^(?:[0-9a-fA-F]{3}){1,2}$/.test(color)) {
-				color = '#' + color;
-			}
-			rules[`mtk${i}`] = `color: ${color};`;
+			rules[`mtk${i}`] = `color: ${color.toRGBHex()};`;
 		}
 		rules['mtki'] = 'font-style: italic;';
 		rules['mtkb'] = 'font-weight: bold;';
 		rules['mtku'] = 'text-decoration: underline;';
 
-		let defaultForegroundColor = /^(?:[0-9a-fA-F]{3}){1,2}$/.test(colorMap[1]) ? '#' + colorMap[1] : colorMap[1];
-		let defaultBackgroundColor = /^(?:[0-9a-fA-F]{3}){1,2}$/.test(colorMap[2]) ? '#' + colorMap[2] : colorMap[2];
+		let defaultForegroundColor = colorMap[1].toRGBHex();
+		let defaultBackgroundColor = colorMap[2].toRGBHex();
 
 		let fontInfo = this.configuration.editor.fontInfo;
 

src/vs/editor/contrib/inspectTMScopes/electron-browser/inspectTMScopes.ts

@@ -23,6 +23,7 @@ import { TokenizationRegistry, LanguageIdentifier, FontStyle, StandardTokenType
 import { CharCode } from 'vs/base/common/charCode';
 import { findMatchingThemeRule } from 'vs/editor/electron-browser/textMate/TMHelper';
 import { IThemeService } from 'vs/workbench/services/themes/common/themeService';
+import { Color } from 'vs/base/common/color';
 
 @editorContribution
 class InspectTMScopesController extends Disposable implements IEditorContribution {
@@ -114,8 +115,8 @@ interface IDecodedMetadata {
 	languageIdentifier: LanguageIdentifier;
 	tokenType: StandardTokenType;
 	fontStyle: FontStyle;
-	foreground: string;
-	background: string;
+	foreground: Color;
+	background: Color;
 }
 
 function renderTokenText(tokenText: string): string {
@@ -237,8 +238,8 @@ class InspectTMScopesWidget extends Disposable implements IContentWidget {
 		result += `<tr><td class="tm-metadata-key">language</td><td class="tm-metadata-value">${escape(metadata.languageIdentifier.language)}</td>`;
 		result += `<tr><td class="tm-metadata-key">token type</td><td class="tm-metadata-value">${this._tokenTypeToString(metadata.tokenType)}</td>`;
 		result += `<tr><td class="tm-metadata-key">font style</td><td class="tm-metadata-value">${this._fontStyleToString(metadata.fontStyle)}</td>`;
-		result += `<tr><td class="tm-metadata-key">foreground</td><td class="tm-metadata-value">${metadata.foreground}</td>`;
-		result += `<tr><td class="tm-metadata-key">background</td><td class="tm-metadata-value">${metadata.background}</td>`;
+		result += `<tr><td class="tm-metadata-key">foreground</td><td class="tm-metadata-value">${metadata.foreground.toRGBAHex()}</td>`;
+		result += `<tr><td class="tm-metadata-key">background</td><td class="tm-metadata-value">${metadata.background.toRGBAHex()}</td>`;
 		result += `</tbody></table>`;
 
 		let theme = this._themeService.getColorTheme();

src/vs/editor/contrib/inspectTokens/browser/inspectTokens.ts

@@ -20,6 +20,7 @@ import { CharCode } from 'vs/base/common/charCode';
 import { IStandaloneColorService } from 'vs/editor/common/services/standaloneColorService';
 import { NULL_STATE, nullTokenize, nullTokenize2 } from 'vs/editor/common/modes/nullMode';
 import { Token } from 'vs/editor/common/core/token';
+import { Color } from 'vs/base/common/color';
 
 @editorContribution
 class InspectTokensController extends Disposable implements IEditorContribution {
@@ -109,8 +110,8 @@ interface IDecodedMetadata {
 	languageIdentifier: LanguageIdentifier;
 	tokenType: StandardTokenType;
 	fontStyle: FontStyle;
-	foreground: string;
-	background: string;
+	foreground: Color;
+	background: Color;
 }
 
 function renderTokenText(tokenText: string): string {
@@ -235,8 +236,8 @@ class InspectTokensWidget extends Disposable implements IContentWidget {
 		result += `<tr><td class="tm-metadata-key">language</td><td class="tm-metadata-value">${escape(metadata.languageIdentifier.language)}</td>`;
 		result += `<tr><td class="tm-metadata-key">token type</td><td class="tm-metadata-value">${this._tokenTypeToString(metadata.tokenType)}</td>`;
 		result += `<tr><td class="tm-metadata-key">font style</td><td class="tm-metadata-value">${this._fontStyleToString(metadata.fontStyle)}</td>`;
-		result += `<tr><td class="tm-metadata-key">foreground</td><td class="tm-metadata-value">${metadata.foreground}</td>`;
-		result += `<tr><td class="tm-metadata-key">background</td><td class="tm-metadata-value">${metadata.background}</td>`;
+		result += `<tr><td class="tm-metadata-key">foreground</td><td class="tm-metadata-value">${metadata.foreground.toRGBHex()}</td>`;
+		result += `<tr><td class="tm-metadata-key">background</td><td class="tm-metadata-value">${metadata.background.toRGBHex()}</td>`;
 		result += `</tbody></table>`;
 
 		result += `<hr/>`;

src/vs/editor/electron-browser/textMate/TMSyntax.ts

@@ -21,7 +21,8 @@ import { grammarsExtPoint, IEmbeddedLanguagesMap, ITMSyntaxExtensionPoint } from
 import { TokenizationResult, TokenizationResult2 } from 'vs/editor/common/core/token';
 import { TokenMetadata } from 'vs/editor/common/model/tokensBinaryEncoding';
 import { nullTokenize2 } from 'vs/editor/common/modes/nullMode';
-import { hexToCSSrgba } from 'vs/base/common/color';
+import { generateTokensCSSForColorMap } from 'vs/editor/common/modes/supports/tokenization';
+import { Color } from 'vs/base/common/color';
 
 export class TMScopeRegistry {
 
@@ -150,23 +151,19 @@ export class MainProcessTextMateSyntax implements ITextMateService {
 		});
 	}
 
-	private static _generateCSS(colorMap: string[]): string {
-		let rules: string[] = [];
+	private static _toColorMap(colorMap: string[]): Color[] {
+		let result: Color[] = [null];
 		for (let i = 1, len = colorMap.length; i < len; i++) {
-			let color = colorMap[i];
-			rules[i] = `.mtk${i} { color: ${hexToCSSrgba(color)}; }`;
+			result[i] = Color.fromHex(colorMap[i]);
 		}
-		rules.push('.mtki { font-style: italic; }');
-		rules.push('.mtkb { font-weight: bold; }');
-		rules.push('.mtku { text-decoration: underline; }');
-		return rules.join('\n');
+		return result;
 	}
 
 	private _updateTheme(): void {
 		let colorTheme = this._themeService.getColorTheme();
 		this._grammarRegistry.setTheme({ name: colorTheme.label, settings: colorTheme.settings });
-		let colorMap = this._grammarRegistry.getColorMap();
-		let cssRules = MainProcessTextMateSyntax._generateCSS(colorMap);
+		let colorMap = MainProcessTextMateSyntax._toColorMap(this._grammarRegistry.getColorMap());
+		let cssRules = generateTokensCSSForColorMap(colorMap);
 		this._styleElement.innerHTML = cssRules;
 		TokenizationRegistry.setColorMap(colorMap);
 	}

src/vs/editor/test/common/core/lineTokens.test.ts

@@ -28,7 +28,7 @@ suite('LineTokens', () => {
 			) >>> 0;
 		}
 
-		return new LineTokens(null, binTokens, text);
+		return new LineTokens(binTokens, text);
 	}
 
 	function createTestLineTokens(): LineTokens {

src/vs/editor/test/common/model/model.line.test.ts

@@ -304,7 +304,7 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => {
 		line.applyEdits(new MarkersTracker(), edits, NO_TAB_SIZE);
 
 		assert.equal(line.text, expectedText);
-		assertLineTokens(line.getTokens(0, []), expectedTokens);
+		assertLineTokens(line.getTokens(0), expectedTokens);
 	}
 
 	test('insertion on empty line', () => {
@@ -315,7 +315,7 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => {
 		line.setTokens(0, new Uint32Array(0));
 
 		line.applyEdits(new MarkersTracker(), [{ startColumn: 1, endColumn: 1, text: 'a', forceMoveMarkers: false }], NO_TAB_SIZE);
-		assertLineTokens(line.getTokens(0, []), [new TestToken(0, 1)]);
+		assertLineTokens(line.getTokens(0), [new TestToken(0, 1)]);
 	});
 
 	test('updates tokens on insertion 1', () => {
@@ -877,7 +877,7 @@ suite('Editor Model - modelLine.split text & tokens', () => {
 
 		assert.equal(line.text, expectedText1);
 		assert.equal(other.text, expectedText2);
-		assertLineTokens(line.getTokens(0, []), expectedTokens);
+		assertLineTokens(line.getTokens(0), expectedTokens);
 	}
 
 	test('split at the beginning', () => {
@@ -963,7 +963,7 @@ suite('Editor Model - modelLine.append text & tokens', () => {
 		a.append(new MarkersTracker(), b, NO_TAB_SIZE);
 
 		assert.equal(a.text, expectedText);
-		assertLineTokens(a.getTokens(0, []), expectedTokens);
+		assertLineTokens(a.getTokens(0), expectedTokens);
 	}
 
 	test('append empty 1', () => {

src/vs/editor/test/common/modesTestUtils.ts

@@ -30,5 +30,5 @@ export function createFakeScopedLineTokens(rawTokens: TokenText[]): ScopedLineTo
 		line += rawToken.text;
 	}
 
-	return createScopedLineTokens(new LineTokens([], tokens, line), 0);
+	return createScopedLineTokens(new LineTokens(tokens, line), 0);
 }

src/vs/workbench/parts/themes/test/electron-browser/themes.test.contribution.ts

@@ -20,6 +20,7 @@ import { IGrammar, StackElement } from 'vscode-textmate';
 import { TokenizationRegistry } from 'vs/editor/common/modes';
 import { TokenMetadata } from 'vs/editor/common/model/tokensBinaryEncoding';
 import { ThemeRule, findMatchingThemeRule } from 'vs/editor/electron-browser/textMate/TMHelper';
+import { Color } from 'vs/base/common/color';
 
 interface IToken {
 	c: string;
@@ -29,7 +30,7 @@ interface IToken {
 
 interface IThemedToken {
 	text: string;
-	color: string;
+	color: Color;
 }
 
 interface IThemesResult {
@@ -56,27 +57,29 @@ class ThemeDocument {
 		}
 	}
 
-	private _generateExplanation(selector: string, color: string): string {
-		return `${selector}: ${color}`;
+	private _generateExplanation(selector: string, color: Color): string {
+		let rgba = color.toRGBA();
+		if (rgba.a === 255) {
+			return `${selector}: ${color.toRGBHex().toUpperCase()}`;
+		}
+		return `${selector}: ${color.toRGBAHex().toUpperCase()}`;
 	}
 
-	public explainTokenColor(scopes: string, color: string): string {
+	public explainTokenColor(scopes: string, color: Color): string {
 
 		let matchingRule = this._findMatchingThemeRule(scopes);
 		if (!matchingRule) {
-			let actual = color.toUpperCase();
-			let expected = this._defaultColor.toUpperCase();
+			let expected = Color.fromHex(this._defaultColor);
 			// No matching rule
-			if (actual !== expected) {
-				throw new Error(`[${this._theme.label}]: Unexpected color ${actual} for ${scopes}. Expected default ${expected}`);
+			if (!color.equals(expected)) {
+				throw new Error(`[${this._theme.label}]: Unexpected color ${color.toRGBAHex()} for ${scopes}. Expected default ${expected.toRGBAHex()}`);
 			}
 			return this._generateExplanation('default', color);
 		}
 
-		let actual = color.toUpperCase();
-		let expected = matchingRule.settings.foreground.toUpperCase();
-		if (actual !== expected) {
-			throw new Error(`[${this._theme.label}]: Unexpected color ${actual} for ${scopes}. Expected ${expected} coming in from ${matchingRule.rawSelector}`);
+		let expected = Color.fromHex(matchingRule.settings.foreground);
+		if (!color.equals(expected)) {
+			throw new Error(`[${this._theme.label}]: Unexpected color ${color.toRGBAHex()} for ${scopes}. Expected ${expected.toRGBAHex()} coming in from ${matchingRule.rawSelector}`);
 		}
 		return this._generateExplanation(matchingRule.rawSelector, color);
 	}
@@ -114,11 +117,10 @@ class Snapper {
 				let tokenText = line.substring(startOffset, endOffset);
 
 				let color = TokenMetadata.getForeground(metadata);
-				let colorStr = colorMap[color];
 
 				result[resultLen++] = {
 					text: tokenText,
-					color: colorStr
+					color: colorMap[color]
 				};
 			}