tokenizer2 APIs for monaco

build/monaco/monaco.d.ts.recipe

@@ -60,6 +60,11 @@ declare namespace monaco {
 
 declare namespace monaco.editor {
 
+	export type ThemeType = 'light' | 'dark' | 'hc';
+	export interface ITheme {
+		readonly type: ThemeType;
+	}
+
 #includeAll(vs/editor/standalone/browser/standaloneEditor;modes.=>languages.;editorCommon.=>):
 #include(vs/editor/standalone/common/standaloneThemeService): BuiltinTheme, IStandaloneThemeData, IColors
 #include(vs/editor/common/modes/supports/tokenization): ITokenThemeRule

src/vs/editor/common/modes/supports/tokenization.ts

@@ -99,7 +99,7 @@ export function parseTokenTheme(source: ITokenThemeRule[]): ParsedTokenThemeRule
 /**
  * Resolve rules (i.e. inheritance).
  */
-function resolveParsedTokenThemeRules(parsedThemeRules: ParsedTokenThemeRule[]): TokenTheme {
+function resolveParsedTokenThemeRules(parsedThemeRules: ParsedTokenThemeRule[], customTokenColors: string[]): TokenTheme {
 
 	// Sort rules lexicographically, and then by index if necessary
 	parsedThemeRules.sort((a, b) => {
@@ -127,9 +127,17 @@ function resolveParsedTokenThemeRules(parsedThemeRules: ParsedTokenThemeRule[]):
 		}
 	}
 	let colorMap = new ColorMap();
+
 	// ensure default foreground gets id 1 and default background gets id 2
-	let defaults = new ThemeTrieElementRule(defaultFontStyle, colorMap.getId(defaultForeground), colorMap.getId(defaultBackground));
+	let foregroundColorId = colorMap.getId(defaultForeground);
+	let backgroundColorId = colorMap.getId(defaultBackground);
+
+	// start with token colors from custom token themes
+	for (let color of customTokenColors) {
+		colorMap.getId(color);
+	}
 
+	let defaults = new ThemeTrieElementRule(defaultFontStyle, foregroundColorId, backgroundColorId);
 	let root = new ThemeTrieElement(defaults);
 	for (let i = 0, len = parsedThemeRules.length; i < len; i++) {
 		let rule = parsedThemeRules[i];
@@ -177,12 +185,12 @@ export class ColorMap {
 
 export class TokenTheme {
 
-	public static createFromRawTokenTheme(source: ITokenThemeRule[]): TokenTheme {
-		return this.createFromParsedTokenTheme(parseTokenTheme(source));
+	public static createFromRawTokenTheme(source: ITokenThemeRule[], customTokenColors: string[]): TokenTheme {
+		return this.createFromParsedTokenTheme(parseTokenTheme(source), customTokenColors);
 	}
 
-	public static createFromParsedTokenTheme(source: ParsedTokenThemeRule[]): TokenTheme {
-		return resolveParsedTokenThemeRules(source);
+	public static createFromParsedTokenTheme(source: ParsedTokenThemeRule[], customTokenColors: string[]): TokenTheme {
+		return resolveParsedTokenThemeRules(source, customTokenColors);
 	}
 
 	private readonly _colorMap: ColorMap;

src/vs/editor/standalone/browser/standaloneEditor.ts

@@ -38,6 +38,8 @@ import { CursorChangeReason } from 'vs/editor/common/controller/cursorEvents';
 import { ITextModel, OverviewRulerLane, EndOfLinePreference, DefaultEndOfLine, EndOfLineSequence, TrackedRangeStickiness, TextModelResolvedOptions, FindMatch } from 'vs/editor/common/model';
 import { INotificationService } from 'vs/platform/notification/common/notification';
 import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
+import { Event } from 'vs/base/common/event';
+import { ITheme } from 'vs/platform/theme/common/themeService';
 
 function withAllStandaloneServices<T extends editorCommon.IEditor>(domElement: HTMLElement, override: IEditorOverrideServices, callback: (services: DynamicStandaloneServices) => T): T {
 	let services = new DynamicStandaloneServices(domElement, override);
@@ -165,8 +167,8 @@ export function createModel(value: string, language?: string, uri?: URI): ITextM
 /**
  * Change the language for a model.
  */
-export function setModelLanguage(model: ITextModel, language: string): void {
-	StaticServices.modelService.get().setMode(model, StaticServices.modeService.get().getOrCreateMode(language));
+export function setModelLanguage(model: ITextModel, languageId: string): void {
+	StaticServices.modelService.get().setMode(model, StaticServices.modeService.get().getOrCreateMode(languageId));
 }
 
 /**
@@ -262,14 +264,14 @@ export function colorizeModelLine(model: ITextModel, lineNumber: number, tabSize
 /**
  * @internal
  */
-function getSafeTokenizationSupport(languageId: string): modes.ITokenizationSupport {
-	let tokenizationSupport = modes.TokenizationRegistry.get(languageId);
+function getSafeTokenizationSupport(language: string): modes.ITokenizationSupport {
+	let tokenizationSupport = modes.TokenizationRegistry.get(language);
 	if (tokenizationSupport) {
 		return tokenizationSupport;
 	}
 	return {
 		getInitialState: () => NULL_STATE,
-		tokenize: (line: string, state: modes.IState, deltaOffset: number) => nullTokenize(languageId, line, state, deltaOffset),
+		tokenize: (line: string, state: modes.IState, deltaOffset: number) => nullTokenize(language, line, state, deltaOffset),
 		tokenize2: undefined,
 	};
 }
@@ -297,12 +299,17 @@ export function tokenize(text: string, languageId: string): Token[][] {
 }
 
 /**
- * Define a new theme.
+ * Define a new theme or updte an existing theme.
  */
 export function defineTheme(themeName: string, themeData: IStandaloneThemeData): void {
 	StaticServices.standaloneThemeService.get().defineTheme(themeName, themeData);
 }
 
+/**
+ * Theme change event.
+ */
+export const onThemeChange: Event<ITheme> = StaticServices.standaloneThemeService.get().onThemeChange;
+
 /**
  * Switches to a theme.
  */
@@ -366,6 +373,7 @@ export function createMonacoEditorAPI(): typeof monaco.editor {
 		tokenize: tokenize,
 		defineTheme: defineTheme,
 		setTheme: setTheme,
+		onThemeChange: onThemeChange,
 
 		// enums
 		ScrollbarVisibility: ScrollbarVisibility,
@@ -394,5 +402,6 @@ export function createMonacoEditorAPI(): typeof monaco.editor {
 
 		// vars
 		EditorType: editorCommon.EditorType
+
 	};
 }

src/vs/editor/standalone/browser/standaloneLanguages.ts

@@ -42,6 +42,11 @@ export function getLanguages(): ILanguageExtensionPoint[] {
 	return result;
 }
 
+export function getLanguageNumericId(languageId: string): number {
+	let lid = StaticServices.modeService.get().getLanguageIdentifier(languageId);
+	return lid && lid.id;
+}
+
 /**
  * An event emitted when a language is first time needed (e.g. a model has it set).
  * @event
@@ -69,6 +74,31 @@ export function setLanguageConfiguration(languageId: string, configuration: Lang
 	return LanguageConfigurationRegistry.register(languageIdentifier, configuration);
 }
 
+/**
+ * @internal
+ */
+export class BinaryTokenizationSupport2Adapter implements modes.ITokenizationSupport {
+
+	private readonly _actual: BinaryTokensProvider;
+
+	constructor(actual: BinaryTokensProvider) {
+		this._actual = actual;
+	}
+
+	public getInitialState(): modes.IState {
+		return this._actual.getInitialState();
+	}
+
+	public tokenize(line: string, state: modes.IState, offsetDelta: number): TokenizationResult {
+		throw new Error('Not supported!');
+	}
+
+	public tokenize2(line: string, state: modes.IState): TokenizationResult2 {
+		let result = this._actual.tokenize2(line, state);
+		return new TokenizationResult2(result.tokens, result.endState);
+	}
+}
+
 /**
  * @internal
  */
@@ -203,6 +233,40 @@ export interface ILineTokens {
 	endState: modes.IState;
 }
 
+/**
+ * The result of a line tokenization.
+ */
+export interface IBinaryLineTokens {
+	/**
+	 * The tokens on the line in binary format. Each token occupies two array indices. For token i:
+	 *  - at offset 2*i => startIndex
+	 *  - at offset 2*i + 1 => metadata
+	 * Meta data is in binary format:
+	 * - -------------------------------------------
+	 *     3322 2222 2222 1111 1111 1100 0000 0000
+	 *     1098 7654 3210 9876 5432 1098 7654 3210
+	 * - -------------------------------------------
+	 *     xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx
+	 *     bbbb bbbb bfff ffff ffFF FTTT LLLL LLLL
+	 * - -------------------------------------------
+	 *  - L = LanguageNumericId (8 bits)
+	 *  - T = StandardTokenType (3 bits)
+	 *  - F = FontStyle (3 bits)
+	 *  - f = foreground colorId (9 bits)
+	 *  - b = background colorId (9 bits)
+	 *
+	 * - Use `getLanguageNumericId` to get the numeric ID of a language.
+	 * - colorIds must be > 0 and are indexes into the `customTokenColors` property in the IStandaloneThemeData:
+	 * - The color value for colorId = 1 is stored in IStandaloneThemeData.customTokenColors[0].
+	 */
+	tokens: Uint32Array;
+	/**
+	 * The tokenization end state.
+	 * A pointer will be held to this and the object should not be modified by the tokenizer after the pointer is returned.
+	 */
+	endState: modes.IState;
+}
+
 /**
  * A "manual" provider of tokens.
  */
@@ -217,6 +281,20 @@ export interface TokensProvider {
 	tokenize(line: string, state: modes.IState): ILineTokens;
 }
 
+/**
+ * A "manual" provider of tokens, returning tokens in a binary form.
+ */
+export interface BinaryTokensProvider {
+	/**
+	 * The initial state of a language. Will be the state passed in to tokenize the first line.
+	 */
+	getInitialState(): modes.IState;
+	/**
+	 * Tokenize a line given the state at the beginning of the line.
+	 */
+	tokenize2(line: string, state: modes.IState): IBinaryLineTokens;
+}
+
 /**
  * Set the tokens provider for a language (manual implementation).
  */
@@ -229,6 +307,18 @@ export function setTokensProvider(languageId: string, provider: TokensProvider):
 	return modes.TokenizationRegistry.register(languageId, adapter);
 }
 
+/**
+ * Set the tokens provider for a language (manual implementation with styled tokens).
+ */
+export function setBinaryTokensProvider(languageId: string, provider: BinaryTokensProvider): IDisposable {
+	let languageIdentifier = StaticServices.modeService.get().getLanguageIdentifier(languageId);
+	if (!languageIdentifier) {
+		throw new Error(`Cannot set tokens provider for unknown language ${languageId}`);
+	}
+
+	return modes.TokenizationRegistry.register(languageId, new BinaryTokenizationSupport2Adapter(provider));
+}
+
 /**
  * Set the tokens provider for a language (monarch implementation).
  */
@@ -764,10 +854,12 @@ export function createMonacoLanguagesAPI(): typeof monaco.languages {
 		register: register,
 		getLanguages: getLanguages,
 		onLanguage: onLanguage,
+		getLanguageNumericId: getLanguageNumericId,
 
 		// provider methods
 		setLanguageConfiguration: setLanguageConfiguration,
 		setTokensProvider: setTokensProvider,
+		setBinaryTokensProvider: setBinaryTokensProvider,
 		setMonarchTokensProvider: setMonarchTokensProvider,
 		registerReferenceProvider: registerReferenceProvider,
 		registerRenameProvider: registerRenameProvider,

src/vs/editor/standalone/browser/standaloneThemeServiceImpl.ts

@@ -5,7 +5,7 @@
 'use strict';
 
 import { TokenTheme, ITokenThemeRule, generateTokensCSSForColorMap } from 'vs/editor/common/modes/supports/tokenization';
-import { IStandaloneThemeService, BuiltinTheme, IStandaloneThemeData, IStandaloneTheme, IColors } from 'vs/editor/standalone/common/standaloneThemeService';
+import { IStandaloneThemeService, BuiltinTheme, IStandaloneThemeData, IStandaloneTheme } from 'vs/editor/standalone/common/standaloneThemeService';
 import { vs, vs_dark, hc_black } from 'vs/editor/standalone/common/themes';
 import * as dom from 'vs/base/browser/dom';
 import { TokenizationRegistry } from 'vs/editor/common/modes';
@@ -26,13 +26,14 @@ const themingRegistry = Registry.as<IThemingRegistry>(ThemingExtensions.ThemingC
 class StandaloneTheme implements IStandaloneTheme {
 	public readonly id: string;
 	public readonly themeName: string;
-	private rules: ITokenThemeRule[];
-	public readonly base: string;
+
+	private themeData: IStandaloneThemeData;
 	private colors: { [colorId: string]: Color };
 	private defaultColors: { [colorId: string]: Color };
 	private _tokenTheme: TokenTheme;
 
-	constructor(base: string, name: string, colors: IColors, rules: ITokenThemeRule[]) {
+	constructor(name: string, standaloneThemeData: IStandaloneThemeData) {
+		let base = standaloneThemeData.base;
 		if (name.length > 0) {
 			this.id = base + ' ' + name;
 			this.themeName = name;
@@ -40,18 +41,46 @@ class StandaloneTheme implements IStandaloneTheme {
 			this.id = base;
 			this.themeName = base;
 		}
-		this.base = base;
-		this.rules = rules;
-		this.colors = {};
-		for (let id in colors) {
-			this.colors[id] = Color.fromHex(colors[id]);
-		}
+		this.colors = null;
 		this.defaultColors = {};
+		this._tokenTheme = null;
+	}
+
+	public get base(): string {
+		return this.themeData.base;
+	}
+
+	public notifyBaseUpdated() {
+		if (this.themeData.inherit) {
+			this.colors = null;
+			this._tokenTheme = null;
+		}
+	}
+
+	private getColors(): { [colorId: string]: Color } {
+		if (!this.colors) {
+			let colors: { [colorId: string]: Color } = Object.create(null);
+			for (let id in this.themeData.colors) {
+				colors[id] = Color.fromHex(this.themeData.colors[id]);
+			}
+			if (this.themeData.inherit) {
+				let baseData = getBuiltinRules(this.themeData.base);
+				for (let id in baseData.colors) {
+					if (!colors[id]) {
+						colors[id] = Color.fromHex(baseData.colors[id]);
+					}
+
+				}
+			}
+			this.colors = colors;
+		}
+		return this.colors;
 	}
 
 	public getColor(colorId: ColorIdentifier, useDefault?: boolean): Color {
-		if (this.colors.hasOwnProperty(colorId)) {
-			return this.colors[colorId];
+		const colors = this.getColors();
+		if (colors.hasOwnProperty(colorId)) {
+			return colors[colorId];
 		}
 		if (useDefault !== false) {
 			return this.getDefault(colorId);
@@ -69,7 +98,7 @@ class StandaloneTheme implements IStandaloneTheme {
 	}
 
 	public defines(colorId: ColorIdentifier): boolean {
-		return this.colors.hasOwnProperty(colorId);
+		return this.getColors().hasOwnProperty(colorId);
 	}
 
 	public get type() {
@@ -82,7 +111,18 @@ class StandaloneTheme implements IStandaloneTheme {
 
 	public get tokenTheme(): TokenTheme {
 		if (!this._tokenTheme) {
-			this._tokenTheme = TokenTheme.createFromRawTokenTheme(this.rules);
+			let rules: ITokenThemeRule[] = [];
+			let customTokenColors = [];
+			if (this.themeData.inherit) {
+				let baseData = getBuiltinRules(this.themeData.base);
+				rules = baseData.rules;
+				customTokenColors = baseData.customTokenColors || [];
+			}
+			rules = rules.concat(this.themeData.rules);
+			if (this.themeData.customTokenColors) {
+				customTokenColors = customTokenColors.concat(this.themeData.customTokenColors);
+			}
+			this._tokenTheme = TokenTheme.createFromRawTokenTheme(rules, customTokenColors);
 		}
 		return this._tokenTheme;
 	}
@@ -109,7 +149,7 @@ function getBuiltinRules(builtinTheme: BuiltinTheme): IStandaloneThemeData {
 
 function newBuiltInTheme(builtinTheme: BuiltinTheme): StandaloneTheme {
 	let themeData = getBuiltinRules(builtinTheme);
-	return new StandaloneTheme(builtinTheme, '', themeData.colors, themeData.rules);
+	return new StandaloneTheme(builtinTheme, themeData);
 }
 
 export class StandaloneThemeServiceImpl implements IStandaloneThemeService {
@@ -139,28 +179,25 @@ export class StandaloneThemeServiceImpl implements IStandaloneThemeService {
 	}
 
 	public defineTheme(themeName: string, themeData: IStandaloneThemeData): void {
-		if (!/^[a-z0-9\-]+$/i.test(themeName) || isBuiltinTheme(themeName)) {
+		if (!/^[a-z0-9\-]+$/i.test(themeName)) {
 			throw new Error('Illegal theme name!');
 		}
-		if (!isBuiltinTheme(themeData.base)) {
+		if (!isBuiltinTheme(themeData.base) && !isBuiltinTheme(themeName)) {
 			throw new Error('Illegal theme base!');
 		}
+		// set or replace theme
+		this._knownThemes.set(themeName, new StandaloneTheme(themeName, themeData));
 
-		let rules: ITokenThemeRule[] = [];
-		let colors: IColors = {};
-		if (themeData.inherit) {
-			let baseData = getBuiltinRules(themeData.base);
-			rules = rules.concat(baseData.rules);
-			for (let id in baseData.colors) {
-				colors[id] = baseData.colors[id];
-			}
+		if (isBuiltinTheme(themeName)) {
+			this._knownThemes.forEach(theme => {
+				if (theme.base === themeName) {
+					theme.notifyBaseUpdated();
+				}
+			});
 		}
-		rules = rules.concat(themeData.rules);
-		for (let id in themeData.colors) {
-			colors[id] = themeData.colors[id];
+		if (this._theme && this._theme.themeName === themeName) {
+			this.setTheme(themeName); // refresh theme
 		}
-
-		this._knownThemes.set(themeName, new StandaloneTheme(themeData.base, themeName, colors, rules));
 	}
 
 	public getTheme(): IStandaloneTheme {

src/vs/editor/standalone/common/standaloneThemeService.ts

@@ -17,6 +17,7 @@ export interface IStandaloneThemeData {
 	base: BuiltinTheme;
 	inherit: boolean;
 	rules: ITokenThemeRule[];
+	customTokenColors?: string[];
 	colors: IColors;
 }
 

src/vs/editor/test/common/modes/supports/tokenization.test.ts

@@ -15,7 +15,7 @@ suite('Token theme matching', () => {
 			{ token: '', foreground: '100000', background: '200000' },
 			{ token: 'punctuation.definition.string.begin.html', foreground: '300000' },
 			{ token: 'punctuation.definition.string', foreground: '400000' },
-		]);
+		], []);
 
 		let colorMap = new ColorMap();
 		colorMap.getId('100000');
@@ -42,7 +42,7 @@ suite('Token theme matching', () => {
 			{ token: 'constant.numeric.oct', fontStyle: 'bold italic underline' },
 			{ token: 'constant.numeric.dec', fontStyle: '', foreground: '500000' },
 			{ token: 'storage.object.bar', fontStyle: '', foreground: '600000' },
-		]);
+		], []);
 
 		let colorMap = new ColorMap();
 		const _A = colorMap.getId('F8F8F2');
@@ -167,7 +167,7 @@ suite('Token theme resolving', () => {
 	});
 
 	test('always has defaults', () => {
-		let actual = TokenTheme.createFromParsedTokenTheme([]);
+		let actual = TokenTheme.createFromParsedTokenTheme([], []);
 		let colorMap = new ColorMap();
 		const _A = colorMap.getId('000000');
 		const _B = colorMap.getId('ffffff');
@@ -178,7 +178,7 @@ suite('Token theme resolving', () => {
 	test('respects incoming defaults 1', () => {
 		let actual = TokenTheme.createFromParsedTokenTheme([
 			new ParsedTokenThemeRule('', -1, FontStyle.NotSet, null, null)
-		]);
+		], []);
 		let colorMap = new ColorMap();
 		const _A = colorMap.getId('000000');
 		const _B = colorMap.getId('ffffff');
@@ -189,7 +189,7 @@ suite('Token theme resolving', () => {
 	test('respects incoming defaults 2', () => {
 		let actual = TokenTheme.createFromParsedTokenTheme([
 			new ParsedTokenThemeRule('', -1, FontStyle.None, null, null)
-		]);
+		], []);
 		let colorMap = new ColorMap();
 		const _A = colorMap.getId('000000');
 		const _B = colorMap.getId('ffffff');
@@ -200,7 +200,7 @@ suite('Token theme resolving', () => {
 	test('respects incoming defaults 3', () => {
 		let actual = TokenTheme.createFromParsedTokenTheme([
 			new ParsedTokenThemeRule('', -1, FontStyle.Bold, null, null)
-		]);
+		], []);
 		let colorMap = new ColorMap();
 		const _A = colorMap.getId('000000');
 		const _B = colorMap.getId('ffffff');
@@ -211,7 +211,7 @@ suite('Token theme resolving', () => {
 	test('respects incoming defaults 4', () => {
 		let actual = TokenTheme.createFromParsedTokenTheme([
 			new ParsedTokenThemeRule('', -1, FontStyle.NotSet, 'ff0000', null)
-		]);
+		], []);
 		let colorMap = new ColorMap();
 		const _A = colorMap.getId('ff0000');
 		const _B = colorMap.getId('ffffff');
@@ -222,7 +222,7 @@ suite('Token theme resolving', () => {
 	test('respects incoming defaults 5', () => {
 		let actual = TokenTheme.createFromParsedTokenTheme([
 			new ParsedTokenThemeRule('', -1, FontStyle.NotSet, null, 'ff0000')
-		]);
+		], []);
 		let colorMap = new ColorMap();
 		const _A = colorMap.getId('000000');
 		const _B = colorMap.getId('ff0000');
@@ -235,7 +235,7 @@ suite('Token theme resolving', () => {
 			new ParsedTokenThemeRule('', -1, FontStyle.NotSet, null, 'ff0000'),
 			new ParsedTokenThemeRule('', -1, FontStyle.NotSet, '00ff00', null),
 			new ParsedTokenThemeRule('', -1, FontStyle.Bold, null, null),
-		]);
+		], []);
 		let colorMap = new ColorMap();
 		const _A = colorMap.getId('00ff00');
 		const _B = colorMap.getId('ff0000');
@@ -247,7 +247,7 @@ suite('Token theme resolving', () => {
 		let actual = TokenTheme.createFromParsedTokenTheme([
 			new ParsedTokenThemeRule('', -1, FontStyle.NotSet, 'F8F8F2', '272822'),
 			new ParsedTokenThemeRule('var', -1, FontStyle.NotSet, 'ff0000', null)
-		]);
+		], []);
 		let colorMap = new ColorMap();
 		const _A = colorMap.getId('F8F8F2');
 		const _B = colorMap.getId('272822');
@@ -264,7 +264,7 @@ suite('Token theme resolving', () => {
 			new ParsedTokenThemeRule('', -1, FontStyle.NotSet, 'F8F8F2', '272822'),
 			new ParsedTokenThemeRule('var', 1, FontStyle.Bold, null, null),
 			new ParsedTokenThemeRule('var', 0, FontStyle.NotSet, 'ff0000', null),
-		]);
+		], []);
 		let colorMap = new ColorMap();
 		const _A = colorMap.getId('F8F8F2');
 		const _B = colorMap.getId('272822');
@@ -281,7 +281,7 @@ suite('Token theme resolving', () => {
 			new ParsedTokenThemeRule('', -1, FontStyle.NotSet, 'F8F8F2', '272822'),
 			new ParsedTokenThemeRule('var', -1, FontStyle.Bold, 'ff0000', null),
 			new ParsedTokenThemeRule('var.identifier', -1, FontStyle.NotSet, '00ff00', null),
-		]);
+		], []);
 		let colorMap = new ColorMap();
 		const _A = colorMap.getId('F8F8F2');
 		const _B = colorMap.getId('272822');
@@ -306,7 +306,7 @@ suite('Token theme resolving', () => {
 			new ParsedTokenThemeRule('constant.numeric.hex', 6, FontStyle.Bold, null, null),
 			new ParsedTokenThemeRule('constant.numeric.oct', 7, FontStyle.Bold | FontStyle.Italic | FontStyle.Underline, null, null),
 			new ParsedTokenThemeRule('constant.numeric.dec', 8, FontStyle.None, '300000', null),
-		]);
+		], []);
 		let colorMap = new ColorMap();
 		const _A = colorMap.getId('F8F8F2');
 		const _B = colorMap.getId('272822');
@@ -330,4 +330,18 @@ suite('Token theme resolving', () => {
 		});
 		assert.deepEqual(actual.getThemeTrieElement(), root);
 	});
+
+	test('custom colors are first in color map', () => {
+		let actual = TokenTheme.createFromParsedTokenTheme([
+			new ParsedTokenThemeRule('var', -1, FontStyle.NotSet, 'F8F8F2', null)
+		], [
+				'000000', 'FFFFFF', '0F0F0F'
+			]);
+		let colorMap = new ColorMap();
+		colorMap.getId('000000');
+		colorMap.getId('FFFFFF');
+		colorMap.getId('0F0F0F');
+		colorMap.getId('F8F8F2');
+		assert.deepEqual(actual.getColorMap(), colorMap.getColorMap());
+	});
 });

src/vs/monaco.d.ts

@@ -776,6 +776,11 @@ declare namespace monaco {
 
 declare namespace monaco.editor {
 
+	export type ThemeType = 'light' | 'dark' | 'hc';
+	export interface ITheme {
+		readonly type: ThemeType;
+	}
+
 
 	/**
 	 * Create a new editor under `domElement`.
@@ -822,7 +827,7 @@ declare namespace monaco.editor {
 	/**
 	 * Change the language for a model.
 	 */
-	export function setModelLanguage(model: ITextModel, language: string): void;
+	export function setModelLanguage(model: ITextModel, languageId: string): void;
 
 	/**
 	 * Set the markers for a model.
@@ -898,10 +903,15 @@ declare namespace monaco.editor {
 	export function tokenize(text: string, languageId: string): Token[][];
 
 	/**
-	 * Define a new theme.
+	 * Define a new theme or updte an existing theme.
 	 */
 	export function defineTheme(themeName: string, themeData: IStandaloneThemeData): void;
 
+	/**
+	 * Theme change event.
+	 */
+	export const onThemeChange: IEvent<ITheme>;
+
 	/**
 	 * Switches to a theme.
 	 */
@@ -913,6 +923,7 @@ declare namespace monaco.editor {
 		base: BuiltinTheme;
 		inherit: boolean;
 		rules: ITokenThemeRule[];
+		customTokenColors?: string[];
 		colors: IColors;
 	}
 
@@ -4040,6 +4051,8 @@ declare namespace monaco.languages {
 	 */
 	export function getLanguages(): ILanguageExtensionPoint[];
 
+	export function getLanguageNumericId(languageId: string): number;
+
 	/**
 	 * An event emitted when a language is first time needed (e.g. a model has it set).
 	 * @event
@@ -4074,6 +4087,40 @@ declare namespace monaco.languages {
 		endState: IState;
 	}
 
+	/**
+	 * The result of a line tokenization.
+	 */
+	export interface IBinaryLineTokens {
+		/**
+		 * The tokens on the line in binary format. Each token occupies two array indices. For token i:
+		 *  - at offset 2*i => startIndex
+		 *  - at offset 2*i + 1 => metadata
+		 * Meta data is in binary format:
+		 * - -------------------------------------------
+		 *     3322 2222 2222 1111 1111 1100 0000 0000
+		 *     1098 7654 3210 9876 5432 1098 7654 3210
+		 * - -------------------------------------------
+		 *     xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx
+		 *     bbbb bbbb bfff ffff ffFF FTTT LLLL LLLL
+		 * - -------------------------------------------
+		 *  - L = LanguageNumericId (8 bits)
+		 *  - T = StandardTokenType (3 bits)
+		 *  - F = FontStyle (3 bits)
+		 *  - f = foreground colorId (9 bits)
+		 *  - b = background colorId (9 bits)
+		 *
+		 * - Use `getLanguageNumericId` to get the numeric ID of a language.
+		 * - colorIds must be > 0 and are indexes into the `customTokenColors` property in the IStandaloneThemeData:
+		 * - The color value for colorId = 1 is stored in IStandaloneThemeData.customTokenColors[0].
+		 */
+		tokens: Uint32Array;
+		/**
+		 * The tokenization end state.
+		 * A pointer will be held to this and the object should not be modified by the tokenizer after the pointer is returned.
+		 */
+		endState: IState;
+	}
+
 	/**
 	 * A "manual" provider of tokens.
 	 */
@@ -4088,11 +4135,30 @@ declare namespace monaco.languages {
 		tokenize(line: string, state: IState): ILineTokens;
 	}
 
+	/**
+	 * A "manual" provider of tokens, returning tokens in a binary form.
+	 */
+	export interface BinaryTokensProvider {
+		/**
+		 * The initial state of a language. Will be the state passed in to tokenize the first line.
+		 */
+		getInitialState(): IState;
+		/**
+		 * Tokenize a line given the state at the beginning of the line.
+		 */
+		tokenize2(line: string, state: IState): IBinaryLineTokens;
+	}
+
 	/**
 	 * Set the tokens provider for a language (manual implementation).
 	 */
 	export function setTokensProvider(languageId: string, provider: TokensProvider): IDisposable;
 
+	/**
+	 * Set the tokens provider for a language (manual implementation with styled tokens).
+	 */
+	export function setBinaryTokensProvider(languageId: string, provider: BinaryTokensProvider): IDisposable;
+
 	/**
 	 * Set the tokens provider for a language (monarch implementation).
 	 */