monaco tokenize2 fixes

src/vs/editor/common/modes/supports/tokenization.ts

@@ -128,15 +128,15 @@ function resolveParsedTokenThemeRules(parsedThemeRules: ParsedTokenThemeRule[],
 	}
 	let colorMap = new ColorMap();
 
-	// ensure default foreground gets id 1 and default background gets id 2
-	let foregroundColorId = colorMap.getId(defaultForeground);
-	let backgroundColorId = colorMap.getId(defaultBackground);
-
 	// start with token colors from custom token themes
 	for (let color of customTokenColors) {
 		colorMap.getId(color);
 	}
 
+
+	let foregroundColorId = colorMap.getId(defaultForeground);
+	let backgroundColorId = colorMap.getId(defaultBackground);
+
 	let defaults = new ThemeTrieElementRule(defaultFontStyle, foregroundColorId, backgroundColorId);
 	let root = new ThemeTrieElement(defaults);
 	for (let i = 0, len = parsedThemeRules.length; i < len; i++) {
@@ -147,6 +147,8 @@ function resolveParsedTokenThemeRules(parsedThemeRules: ParsedTokenThemeRule[],
 	return new TokenTheme(colorMap, root);
 }
 
+const colorRegExp = /^#?([0-9A-Fa-f]{6})([0-9A-Fa-f]{2})?$/;
+
 export class ColorMap {
 
 	private _lastColorId: number;
@@ -163,10 +165,11 @@ export class ColorMap {
 		if (color === null) {
 			return 0;
 		}
-		color = color.toUpperCase();
-		if (!/^[0-9A-F]{6}$/.test(color)) {
-			throw new Error('Illegal color name: ' + color);
+		const match = color.match(colorRegExp);
+		if (!match) {
+			throw new Error('Illegal value for token color: ' + color);
 		}
+		color = match[1].toUpperCase();
 		let value = this._color2id.get(color);
 		if (value) {
 			return value;

src/vs/editor/standalone/browser/standaloneThemeServiceImpl.ts

@@ -33,6 +33,7 @@ class StandaloneTheme implements IStandaloneTheme {
 	private _tokenTheme: TokenTheme;
 
 	constructor(name: string, standaloneThemeData: IStandaloneThemeData) {
+		this.themeData = standaloneThemeData;
 		let base = standaloneThemeData.base;
 		if (name.length > 0) {
 			this.id = base + ' ' + name;
@@ -42,7 +43,7 @@ class StandaloneTheme implements IStandaloneTheme {
 			this.themeName = base;
 		}
 		this.colors = null;
-		this.defaultColors = {};
+		this.defaultColors = Object.create(null);
 		this._tokenTheme = null;
 	}
 
@@ -78,9 +79,9 @@ class StandaloneTheme implements IStandaloneTheme {
 	}
 
 	public getColor(colorId: ColorIdentifier, useDefault?: boolean): Color {
-		const colors = this.getColors();
-		if (colors.hasOwnProperty(colorId)) {
-			return colors[colorId];
+		const color = this.getColors()[colorId];
+		if (color) {
+			return color;
 		}
 		if (useDefault !== false) {
 			return this.getDefault(colorId);
@@ -89,10 +90,11 @@ class StandaloneTheme implements IStandaloneTheme {
 	}
 
 	private getDefault(colorId: ColorIdentifier): Color {
-		if (this.defaultColors.hasOwnProperty(colorId)) {
-			return this.defaultColors[colorId];
+		let color = this.defaultColors[colorId];
+		if (color) {
+			return color;
 		}
-		let color = colorRegistry.resolveDefaultColor(colorId, this);
+		color = colorRegistry.resolveDefaultColor(colorId, this);
 		this.defaultColors[colorId] = color;
 		return color;
 	}
@@ -116,11 +118,13 @@ class StandaloneTheme implements IStandaloneTheme {
 			if (this.themeData.inherit) {
 				let baseData = getBuiltinRules(this.themeData.base);
 				rules = baseData.rules;
-				customTokenColors = baseData.customTokenColors || [];
+				if (baseData.customTokenColors) {
+					customTokenColors = baseData.customTokenColors;
+				}
 			}
 			rules = rules.concat(this.themeData.rules);
 			if (this.themeData.customTokenColors) {
-				customTokenColors = customTokenColors.concat(this.themeData.customTokenColors);
+				customTokenColors = this.themeData.customTokenColors;
 			}
 			this._tokenTheme = TokenTheme.createFromRawTokenTheme(rules, customTokenColors);
 		}