Begin phasing out usage of tokenization brackets

src/vs/editor/common/editorCommon.ts

@@ -1171,16 +1171,13 @@ export interface ITokensInflatorMap {
 export interface ILineTokensBinaryEncoding {
 	START_INDEX_MASK: number;
 	TYPE_MASK: number;
-	BRACKET_MASK: number;
 	START_INDEX_OFFSET: number;
 	TYPE_OFFSET: number;
-	BRACKET_OFFSET: number;
 
 	deflateArr(map:ITokensInflatorMap, tokens:Modes.IToken[]): number[];
 	inflate(map:ITokensInflatorMap, binaryEncodedToken:number): Modes.IToken;
 	getStartIndex(binaryEncodedToken:number): number;
 	getType(map:ITokensInflatorMap, binaryEncodedToken:number): string;
-	getBracket(binaryEncodedToken:number): Modes.Bracket;
 	inflateArr(map:ITokensInflatorMap, binaryEncodedTokens:number[]): Modes.IToken[];
 	findIndexOfOffset(binaryEncodedTokens:number[], offset:number): number;
 	sliceAndInflate(map:ITokensInflatorMap, binaryEncodedTokens:number[], startOffset:number, endOffset:number, deltaStartIndex:number): Modes.IToken[];
@@ -1204,7 +1201,6 @@ export interface ILineTokens {
 	getTokenCount(): number;
 	getTokenStartIndex(tokenIndex:number): number;
 	getTokenType(tokenIndex:number): string;
-	getTokenBracket(tokenIndex:number): Modes.Bracket;
 	getTokenEndIndex(tokenIndex:number, textLength:number): number;
 
 	/**

src/vs/editor/common/model/modelLine.ts

@@ -171,9 +171,8 @@ export class ModelLine {
 				if (currentTokenStartIndex > 0 && delta !== 0) {
 					// adjust token's `startIndex` by `delta`
 					let deflatedType = (tokens[tokensIndex] / BIN.TYPE_OFFSET) & BIN.TYPE_MASK;
-					let deflatedBracket = (tokens[tokensIndex] / BIN.BRACKET_OFFSET) & BIN.BRACKET_MASK;
 					let newStartIndex = Math.max(minimumAllowedIndex, currentTokenStartIndex + delta);
-					let newToken = deflatedBracket * BIN.BRACKET_OFFSET + deflatedType * BIN.TYPE_OFFSET + newStartIndex * BIN.START_INDEX_OFFSET;
+					let newToken = deflatedType * BIN.TYPE_OFFSET + newStartIndex * BIN.START_INDEX_OFFSET;
 
 					if (delta < 0) {
 						// pop all previous tokens that have become `collapsed`
@@ -439,9 +438,8 @@ export class ModelLine {
 
 					let deflatedStartIndex = (token / BIN.START_INDEX_OFFSET) & BIN.START_INDEX_MASK;
 					let deflatedType = (token / BIN.TYPE_OFFSET) & BIN.TYPE_MASK;
-					let deflatedBracket = (token / BIN.BRACKET_OFFSET) & BIN.BRACKET_MASK;
 					let newStartIndex = deflatedStartIndex + thisTextLength;
-					let newToken = deflatedBracket * BIN.BRACKET_OFFSET + deflatedType * BIN.TYPE_OFFSET + newStartIndex * BIN.START_INDEX_OFFSET;
+					let newToken = deflatedType * BIN.TYPE_OFFSET + newStartIndex * BIN.START_INDEX_OFFSET;
 
 					otherTokens[i] = newToken;
 				}
@@ -618,7 +616,7 @@ function toLineTokens(map:EditorCommon.ITokensInflatorMap, tokens:Modes.IToken[]
 				return null;
 			}
 		} else {
-			if (tokens[0].startIndex === 0 && tokens[0].type === '' && !tokens[0].bracket) {
+			if (tokens[0].startIndex === 0 && tokens[0].type === '') {
 				return null;
 			}
 		}
@@ -628,7 +626,6 @@ function toLineTokens(map:EditorCommon.ITokensInflatorMap, tokens:Modes.IToken[]
 
 var getStartIndex = EditorCommon.LineTokensBinaryEncoding.getStartIndex;
 var getType = EditorCommon.LineTokensBinaryEncoding.getType;
-var getBracket = EditorCommon.LineTokensBinaryEncoding.getBracket;
 var findIndexOfOffset = EditorCommon.LineTokensBinaryEncoding.findIndexOfOffset;
 
 export class LineTokens implements EditorCommon.ILineTokens {
@@ -669,10 +666,6 @@ export class LineTokens implements EditorCommon.ILineTokens {
 		return getType(this.map, this._tokens[tokenIndex]);
 	}
 
-	public getTokenBracket(tokenIndex:number): Modes.Bracket {
-		return getBracket(this._tokens[tokenIndex]);
-	}
-
 	public getTokenEndIndex(tokenIndex:number, textLength:number): number {
 		if (tokenIndex + 1 < this._tokens.length) {
 			return getStartIndex(this._tokens[tokenIndex + 1]);
@@ -714,10 +707,6 @@ class EmptyLineTokens implements EditorCommon.ILineTokens {
 		return Strings.empty;
 	}
 
-	public getTokenBracket(tokenIndex:number): Modes.Bracket {
-		return 0;
-	}
-
 	public getTokenEndIndex(tokenIndex:number, textLength:number): number {
 		return 0;
 	}
@@ -756,10 +745,6 @@ export class DefaultLineTokens implements EditorCommon.ILineTokens {
 		return Strings.empty;
 	}
 
-	public getTokenBracket(tokenIndex:number): Modes.Bracket {
-		return 0;
-	}
-
 	public getTokenEndIndex(tokenIndex:number, textLength:number): number {
 		return textLength;
 	}

src/vs/editor/common/model/textModelWithTokens.ts

@@ -166,10 +166,6 @@ class LineContext implements Modes.ILineContext {
 		return this._lineTokens.getTokenType(tokenIndex);
 	}
 
-	public getTokenBracket(tokenIndex:number): Modes.Bracket {
-		return this._lineTokens.getTokenBracket(tokenIndex);
-	}
-
 	public getTokenText(tokenIndex:number): string {
 		var startIndex = this._lineTokens.getTokenStartIndex(tokenIndex);
 		var endIndex = this._lineTokens.getTokenEndIndex(tokenIndex, this._text.length);

src/vs/editor/common/model/textModelWithTokensHelpers.ts

@@ -10,6 +10,7 @@ import Modes = require('vs/editor/common/modes');
 import EditorCommon = require('vs/editor/common/editorCommon');
 import {Arrays} from 'vs/editor/common/core/arrays';
 import Errors = require('vs/base/common/errors');
+import {getBracketFor} from 'vs/editor/common/modes/supports';
 
 export interface ITextSource {
 
@@ -29,7 +30,6 @@ export interface ITextSource {
 }
 
 var getType = EditorCommon.LineTokensBinaryEncoding.getType;
-var getBracket = EditorCommon.LineTokensBinaryEncoding.getBracket;
 var getStartIndex = EditorCommon.LineTokensBinaryEncoding.getStartIndex;
 
 export interface INonWordTokenMap {
@@ -241,24 +241,23 @@ export class BracketsHelper {
 
 	private static _findMatchingBracketUp(textSource:ITextSource, type:string, lineNumber:number, tokenIndex:number, initialCount:number): Range {
 
-		var i:number,
-			end:number,
-			j:number,
-			count = initialCount;
+		let mode = textSource.getMode();
+		let count = initialCount;
 
-		for (i = lineNumber; i >= 1; i--) {
+		for (let i = lineNumber; i >= 1; i--) {
 
-			var lineTokens = textSource.getLineTokens(i, false),
-				tokens = lineTokens.getBinaryEncodedTokens(),
-				tokensMap = lineTokens.getBinaryEncodedTokensMap(),
-				lineText = textSource.getLineContent(i);
+			let lineTokens = textSource.getLineTokens(i, false);
+			let tokens = lineTokens.getBinaryEncodedTokens();
+			let tokensMap = lineTokens.getBinaryEncodedTokensMap();
+			let lineText = textSource.getLineContent(i);
 
-			for (j = (i === lineNumber ? tokenIndex : tokens.length) - 1; j >= 0; j--) {
+			for (let j = (i === lineNumber ? tokenIndex : tokens.length) - 1; j >= 0; j--) {
 				if (getType(tokensMap, tokens[j]) === type) {
-					count += BracketsHelper._sign(getBracket(tokens[j]));
+					let start = getStartIndex(tokens[j]);
+					let end = (j === tokens.length - 1 ? lineText.length : getStartIndex(tokens[j + 1]));
+					count += BracketsHelper._sign(getBracketFor(type, lineText.substring(start, end), mode));
 					if (count === 0) {
-						end = (j === tokens.length - 1 ? lineText.length : getStartIndex(tokens[j + 1]));
-						return new Range(i, getStartIndex(tokens[j]) + 1, i, end + 1);
+						return new Range(i, start + 1, i, end + 1);
 					}
 				}
 			}
@@ -268,14 +267,10 @@ export class BracketsHelper {
 
 	private static _findMatchingBracketDown(textSource:ITextSource, type:string, lineNumber:number, tokenIndex:number, inaccurateResultAcceptable:boolean): { range:Range; isAccurate:boolean; } {
 
-		var i:number,
-			len:number,
-			end:number,
-			j:number,
-			lenJ:number,
-			count = 1;
+		let mode = textSource.getMode();
+		let count = 1;
 
-		for (i = lineNumber, len = textSource.getLineCount(); i <= len; i++) {
+		for (let i = lineNumber, len = textSource.getLineCount(); i <= len; i++) {
 			if (inaccurateResultAcceptable && !textSource._lineIsTokenized(i)) {
 				return {
 					range: null,
@@ -288,13 +283,14 @@ export class BracketsHelper {
 				tokensMap = lineTokens.getBinaryEncodedTokensMap(),
 				lineText = textSource.getLineContent(i);
 
-			for (j = (i === lineNumber ? tokenIndex + 1 : 0), lenJ = tokens.length; j < lenJ; j++) {
+			for (let j = (i === lineNumber ? tokenIndex + 1 : 0), lenJ = tokens.length; j < lenJ; j++) {
 				if (getType(tokensMap, tokens[j]) === type) {
-					count += BracketsHelper._sign(getBracket(tokens[j]));
+					let start = getStartIndex(tokens[j]);
+					let end = (j === tokens.length - 1 ? lineText.length : getStartIndex(tokens[j + 1]));
+					count += BracketsHelper._sign(getBracketFor(type, lineText.substring(start, end), mode));
 					if (count === 0) {
-						end = (j === tokens.length - 1 ? lineText.length : getStartIndex(tokens[j + 1]));
 						return {
-							range: new Range(i, getStartIndex(tokens[j]) + 1, i, end + 1),
+							range: new Range(i, start + 1, i, end + 1),
 							isAccurate: true
 						};
 					}
@@ -332,6 +328,9 @@ export class BracketsHelper {
 	}
 
 	public static matchBracket(textSource:ITextSource, position:EditorCommon.IPosition, inaccurateResultAcceptable:boolean): EditorCommon.IMatchBracketResult {
+
+		let mode = textSource.getMode();
+
 		if (inaccurateResultAcceptable && !textSource._lineIsTokenized(position.lineNumber)) {
 			return {
 				brackets: null,
@@ -364,9 +363,8 @@ export class BracketsHelper {
 				token = tokens[i];
 				tokenStartIndex = getStartIndex(token);
 				tokenType = getType(tokensMap, token);
-				tokenBracket = getBracket(token);
-
 				end = i === len - 1 ? lineText.length : getStartIndex(tokens[i + 1]);
+				tokenBracket = getBracketFor(tokenType, lineText.substring(tokenStartIndex, end), mode);
 
 				if (tokenStartIndex <= columnIndex && columnIndex <= end) {
 					if (tokenBracket < 0) {

src/vs/editor/common/model/tokensBinaryEncoding.ts

@@ -12,30 +12,25 @@ import Errors = require('vs/base/common/errors');
 class InflatedToken implements Modes.IToken {
 	startIndex:number;
 	type:string;
-	bracket:Modes.Bracket;
 
-	constructor(startIndex:number, type:string, bracket:Modes.Bracket) {
+	constructor(startIndex:number, type:string) {
 		this.startIndex = startIndex;
 		this.type = type;
-		this.bracket = bracket;
 	}
 
 	public toString(): string {
-		return '{ ' + this.startIndex + ', \'' + this.type + '\', ' + this.bracket + '}';
+		return '{ ' + this.startIndex + ', \'' + this.type + '\'}';
 	}
 }
 
 export var START_INDEX_MASK = 0xffffffff;
 export var TYPE_MASK = 0xffff;
-export var BRACKET_MASK = 0xff;
 export var START_INDEX_OFFSET = 1;
 export var TYPE_OFFSET = Math.pow(2, 32);
-export var BRACKET_OFFSET = Math.pow(2, 48);
 
 var DEFAULT_TOKEN = {
 	startIndex: 0,
-	type: '',
-	bracket: 0
+	type: ''
 };
 var INFLATED_TOKENS_EMPTY_TEXT = <Modes.IToken[]>[];
 var DEFLATED_TOKENS_EMPTY_TEXT = <number[]>[];
@@ -46,14 +41,13 @@ export function deflateArr(map:EditorCommon.ITokensInflatorMap, tokens:Modes.ITo
 	if (tokens.length === 0) {
 		return DEFLATED_TOKENS_EMPTY_TEXT;
 	}
-	if (tokens.length === 1 && tokens[0].startIndex === 0 && !tokens[0].type && !tokens[0].bracket) {
+	if (tokens.length === 1 && tokens[0].startIndex === 0 && !tokens[0].type) {
 		return DEFLATED_TOKENS_NON_EMPTY_TEXT;
 	}
 
 	var i:number,
 		len:number,
 		deflatedToken:number,
-		deflatedBracket:number,
 		deflated:number,
 		token:Modes.IToken,
 		inflateMap = map._inflate,
@@ -80,11 +74,6 @@ export function deflateArr(map:EditorCommon.ITokensInflatorMap, tokens:Modes.ITo
 			inflateMap.push(token.type);
 		}
 
-		deflatedBracket = token.bracket;
-		if (deflatedBracket < 0) {
-			deflatedBracket = 2;
-		}
-
 		// http://stackoverflow.com/a/2803010
 		// All numbers in JavaScript are actually IEEE-754 compliant floating-point doubles.
 		// These have a 53-bit mantissa which should mean that any integer value with a magnitude
@@ -96,10 +85,9 @@ export function deflateArr(map:EditorCommon.ITokensInflatorMap, tokens:Modes.ITo
 
 		// 32 bits for startIndex => up to 2^32 = 4,294,967,296
 		// 16 bits for token => up to 2^16 = 65,536
-		// 2 bits for bracket => up to 2^2 = 4
 
-		// [bracket][token][startIndex]
-		deflated = deflatedBracket * BRACKET_OFFSET + deflatedToken * TYPE_OFFSET + token.startIndex * START_INDEX_OFFSET;
+		// [token][startIndex]
+		deflated = deflatedToken * TYPE_OFFSET + token.startIndex * START_INDEX_OFFSET;
 
 		result[i] = deflated;
 
@@ -116,13 +104,8 @@ export function inflate(map:EditorCommon.ITokensInflatorMap, binaryEncodedToken:
 
 	var startIndex = (binaryEncodedToken / START_INDEX_OFFSET) & START_INDEX_MASK;
 	var deflatedType = (binaryEncodedToken / TYPE_OFFSET) & TYPE_MASK;
-	var deflatedBracket = (binaryEncodedToken / BRACKET_OFFSET) & BRACKET_MASK;
-
-	if (deflatedBracket === 2) {
-		deflatedBracket = -1;
-	}
 
-	return new InflatedToken(startIndex, map._inflate[deflatedType], deflatedBracket);
+	return new InflatedToken(startIndex, map._inflate[deflatedType]);
 }
 
 export function getStartIndex(binaryEncodedToken:number): number {
@@ -137,16 +120,6 @@ export function getType(map:EditorCommon.ITokensInflatorMap, binaryEncodedToken:
 	return map._inflate[deflatedType];
 }
 
-export function getBracket(binaryEncodedToken:number): Modes.Bracket {
-	var deflatedBracket = (binaryEncodedToken / BRACKET_OFFSET) & BRACKET_MASK;
-
-	if (deflatedBracket === 2) {
-		deflatedBracket = -1;
-	}
-
-	return deflatedBracket;
-}
-
 export function inflateArr(map:EditorCommon.ITokensInflatorMap, binaryEncodedTokens:number[]): Modes.IToken[] {
 	if (binaryEncodedTokens.length === 0) {
 		return INFLATED_TOKENS_EMPTY_TEXT;
@@ -160,7 +133,6 @@ export function inflateArr(map:EditorCommon.ITokensInflatorMap, binaryEncodedTok
 		len:number,
 		deflated:number,
 		startIndex:number,
-		deflatedBracket:number,
 		deflatedType:number,
 		inflateMap = map._inflate;
 
@@ -169,13 +141,8 @@ export function inflateArr(map:EditorCommon.ITokensInflatorMap, binaryEncodedTok
 
 		startIndex = (deflated / START_INDEX_OFFSET) & START_INDEX_MASK;
 		deflatedType = (deflated / TYPE_OFFSET) & TYPE_MASK;
-		deflatedBracket = (deflated / BRACKET_OFFSET) & BRACKET_MASK;
-
-		if (deflatedBracket === 2) {
-			deflatedBracket = -1;
-		}
 
-		result[i] = new InflatedToken(startIndex, inflateMap[deflatedType], deflatedBracket);
+		result[i] = new InflatedToken(startIndex, inflateMap[deflatedType]);
 	}
 
 	return result;
@@ -200,15 +167,13 @@ export function sliceAndInflate(map:EditorCommon.ITokensInflatorMap, binaryEncod
 		originalStartIndex:number,
 		newStartIndex:number,
 		deflatedType:number,
-		deflatedBracket:number,
 		result: Modes.IToken[] = [],
 		inflateMap = map._inflate;
 
 	originalToken = binaryEncodedTokens[startIndex];
 	deflatedType = (originalToken / TYPE_OFFSET) & TYPE_MASK;
-	deflatedBracket = (originalToken / BRACKET_OFFSET) & BRACKET_MASK;
 	newStartIndex = 0;
-	result.push(new InflatedToken(newStartIndex, inflateMap[deflatedType], deflatedBracket));
+	result.push(new InflatedToken(newStartIndex, inflateMap[deflatedType]));
 
 	for (i = startIndex + 1, len = binaryEncodedTokens.length; i < len; i++) {
 		originalToken = binaryEncodedTokens[i];
@@ -219,9 +184,8 @@ export function sliceAndInflate(map:EditorCommon.ITokensInflatorMap, binaryEncod
 		}
 
 		deflatedType = (originalToken / TYPE_OFFSET) & TYPE_MASK;
-		deflatedBracket = (originalToken / BRACKET_OFFSET) & BRACKET_MASK;
 		newStartIndex = originalStartIndex - startOffset + deltaStartIndex;
-		result.push(new InflatedToken(newStartIndex, inflateMap[deflatedType], deflatedBracket));
+		result.push(new InflatedToken(newStartIndex, inflateMap[deflatedType]));
 	}
 
 	return result;

src/vs/editor/common/modes.ts

@@ -176,7 +176,6 @@ export interface ILineContext {
 	getTokenCount(): number;
 	getTokenStartIndex(tokenIndex:number): number;
 	getTokenType(tokenIndex:number): string;
-	getTokenBracket(tokenIndex:number): Bracket;
 	getTokenText(tokenIndex:number): string;
 	getTokenEndIndex(tokenIndex:number): number;
 	findIndexOfOffset(offset:number): number;
@@ -332,7 +331,7 @@ export interface IMode {
 export interface IToken {
 	startIndex:number;
 	type:string;
-	bracket:Bracket;
+	bracket?:Bracket;
 }
 
 export interface IModeTransition {

src/vs/editor/common/modes/nullMode.ts

@@ -103,8 +103,7 @@ export function nullTokenize(mode: Modes.IMode, buffer:string, state: Modes.ISta
 	var tokens:Modes.IToken[] = [
 		{
 			startIndex: deltaOffset,
-			type: '',
-			bracket: Modes.Bracket.None
+			type: ''
 		}
 	];
 

src/vs/editor/common/modes/supports.ts

@@ -21,16 +21,14 @@ import {IDisposable} from 'vs/base/common/lifecycle';
 export class Token implements Modes.IToken {
 	public startIndex:number;
 	public type:string;
-	public bracket:Modes.Bracket;
 
-	constructor(startIndex:number, type:string, bracket:Modes.Bracket) {
+	constructor(startIndex:number, type:string) {
 		this.startIndex = startIndex;
 		this.type = type;
-		this.bracket = bracket;
 	}
 
 	public toString(): string {
-		return '(' + this.startIndex + ', ' + this.type + ', ' + this.bracket + ')';
+		return '(' + this.startIndex + ', ' + this.type + ')';
 	}
 }
 
@@ -146,10 +144,6 @@ export class FilteredLineContext implements Modes.ILineContext {
 		return this._actual.getTokenType(tokenIndex + this._firstTokenInModeIndex);
 	}
 
-	public getTokenBracket(tokenIndex:number): Modes.Bracket {
-		return this._actual.getTokenBracket(tokenIndex + this._firstTokenInModeIndex);
-	}
-
 	public getTokenText(tokenIndex:number): string {
 		return this._actual.getTokenText(tokenIndex + this._firstTokenInModeIndex);
 	}
@@ -357,7 +351,6 @@ export class TokenizationSupport extends AbstractSupport implements Modes.IToken
 		}
 
 		var maxPos = Math.min(stopAtOffset - deltaOffset, buffer.length);
-		var noneBracket = Modes.Bracket.None;
 		while (lineStream.pos() < maxPos) {
 			beforeTokenizeStreamPos = lineStream.pos();
 
@@ -379,7 +372,7 @@ export class TokenizationSupport extends AbstractSupport implements Modes.IToken
 			} while (!tokenResult.type && tokenResult.type !== '');
 
 			if (previousType !== tokenResult.type || tokenResult.bracket || previousType === null) {
-				prependTokens.push(new Token(beforeTokenizeStreamPos + deltaOffset, tokenResult.type, tokenResult.bracket || noneBracket));
+				prependTokens.push(new Token(beforeTokenizeStreamPos + deltaOffset, tokenResult.type));
 			}
 
 			previousType = tokenResult.type;
@@ -561,7 +554,16 @@ export class BracketElectricCharacterSupport extends AbstractSupport implements
 	}
 }
 
-
+// TODO@Alex -> refactor to use `brackets` from language configuration
+export function getBracketFor(tokenType:string, tokenText:string, mode:Modes.IMode): Modes.Bracket {
+	if (tokenText === '{' || tokenText === '(' || tokenText === '[') {
+		return Modes.Bracket.Open;
+	}
+	if (tokenText === '}' || tokenText === ')' || tokenText === ']') {
+		return Modes.Bracket.Close;
+	}
+	return Modes.Bracket.None;
+}
 
 export interface IDeclarationContribution {
 	tokens?: string[];

src/vs/editor/contrib/smartSelect/common/tokenTree.ts

@@ -8,6 +8,7 @@ import EditorCommon = require('vs/editor/common/editorCommon');
 import Modes = require('vs/editor/common/modes');
 import {Range} from 'vs/editor/common/core/range';
 import {Position} from 'vs/editor/common/core/position';
+import {getBracketFor} from 'vs/editor/common/modes/supports';
 
 export class Node {
 
@@ -108,6 +109,7 @@ class TokenScanner {
 	private _currentLineNumber: number;
 	private _currentTokenIndex: number;
 	private _currentLineTokens: EditorCommon.ILineTokens;
+	private _currentLineText: string;
 
 	constructor(model: EditorCommon.IModel) {
 		this._model = model;
@@ -127,22 +129,28 @@ class TokenScanner {
 		if (!this._currentLineTokens) {
 			// no tokens for this line
 			this._currentLineTokens = this._model.getLineTokens(this._currentLineNumber);
+			this._currentLineText = this._model.getLineContent(this._currentLineNumber);
 			this._currentTokenIndex = 0;
 		}
 		if (this._currentTokenIndex >= this._currentLineTokens.getTokenCount()) {
 			// last token of line visited
 			this._currentLineNumber += 1;
 			this._currentLineTokens = null;
+			this._currentLineText = null;
 			return this.next();
 		}
+		let tokenType = this._currentLineTokens.getTokenType(this._currentTokenIndex);
+		let tokenStartIndex = this._currentLineTokens.getTokenStartIndex(this._currentTokenIndex);
+		let tokenEndIndex = this._currentLineTokens.getTokenEndIndex(this._currentTokenIndex, this._currentLineText.length + 1);
+		let tokenText = this._currentLineText.substring(tokenStartIndex, tokenEndIndex);
 		var token: Token = {
-			type: this._currentLineTokens.getTokenType(this._currentTokenIndex),
-			bracket: this._currentLineTokens.getTokenBracket(this._currentTokenIndex),
+			type: tokenType,
+			bracket: getBracketFor(tokenType, tokenText, this._model.getMode()),
 			range: {
 				startLineNumber: this._currentLineNumber,
-				startColumn: 1 + this._currentLineTokens.getTokenStartIndex(this._currentTokenIndex),
+				startColumn: 1 + tokenStartIndex,
 				endLineNumber: this._currentLineNumber,
-				endColumn: 1 + this._currentLineTokens.getTokenEndIndex(this._currentTokenIndex, this._model.getLineMaxColumn(this._currentLineNumber))
+				endColumn: 1 + tokenEndIndex
 			}
 		};
 		//		token.__debugContent = this._model.getValueInRange(token.range);

src/vs/editor/node/textMate/TMSyntax.ts

@@ -162,10 +162,6 @@ class Tokenizer {
 			modeTransitions: [{ startIndex: offsetDelta, mode: freshState.getMode() }],
 		};
 
-		let noBracket = Modes.Bracket.None,
-			openBracket = Modes.Bracket.Open,
-			closeBracket = Modes.Bracket.Close;
-
 		for (let tokenIndex = 0, len = textMateResult.tokens.length; tokenIndex < len; tokenIndex++) {
 			let token = textMateResult.tokens[tokenIndex];
 			let tokenStartIndex = token.startIndex;
@@ -175,7 +171,7 @@ class Tokenizer {
 
 			if (t.isOpaqueToken) {
 				// Should not do any smartness to detect brackets on this token
-				ret.tokens.push(new supports.Token(tokenStartIndex + offsetDelta, t.tokenType, noBracket));
+				ret.tokens.push(new supports.Token(tokenStartIndex + offsetDelta, t.tokenType));
 				continue;
 			}
 
@@ -187,51 +183,44 @@ class Tokenizer {
 			for (i = tokenStartIndex; i < tokenEndIndex; i++) {
 				charCode = line.charCodeAt(i);
 				isBracket = null;
-				bracketType = noBracket;
 
 				switch (charCode) {
 					case _openParen: // (
 						isBracket = 'delimiter.paren';
-						bracketType = openBracket;
 						break;
 					case _closeParen: // )
 						isBracket = 'delimiter.paren';
-						bracketType = closeBracket;
 						break;
 					case _openCurly: // {
 						isBracket = 'delimiter.curly';
-						bracketType = openBracket;
 						break;
 					case _closeCurly: // }
 						isBracket = 'delimiter.curly';
-						bracketType = closeBracket;
 						break;
 					case _openSquare: // [
 						isBracket = 'delimiter.square';
-						bracketType = openBracket;
 						break;
 					case _closeSquare: // ]
 						isBracket = 'delimiter.square';
-						bracketType = closeBracket;
 						break;
 				}
 
 				if (isBracket) {
 					if (tokenStartIndex < i) {
 						// push a token before character `i`
-						ret.tokens.push(new supports.Token(tokenStartIndex + offsetDelta, t.tokenType, noBracket));
+						ret.tokens.push(new supports.Token(tokenStartIndex + offsetDelta, t.tokenType));
 						tokenStartIndex = i;
 					}
 
 					// push character `i` as a token
-					ret.tokens.push(new supports.Token(tokenStartIndex + offsetDelta, isBracket + '.' + t.modeToken, bracketType));
+					ret.tokens.push(new supports.Token(tokenStartIndex + offsetDelta, isBracket + '.' + t.modeToken));
 					tokenStartIndex = i + 1;
 				}
 			}
 
 			if (tokenStartIndex < tokenEndIndex) {
 				// push the remaining text as a token
-				ret.tokens.push(new supports.Token(tokenStartIndex + offsetDelta, t.tokenType, noBracket));
+				ret.tokens.push(new supports.Token(tokenStartIndex + offsetDelta, t.tokenType));
 			}
 		}
 

src/vs/editor/test/common/modes/tokenization.test.ts

@@ -177,7 +177,6 @@ function assertTokens(actual:modes.IToken[], expected:ITestToken[], message?:str
 	for (var i = 0; i < expected.length; i++) {
 		assert.equal(actual[i].startIndex, expected[i].startIndex, 'startIndex mismatch');
 		assert.equal(actual[i].type, expected[i].type, 'type mismatch');
-		assert.equal(actual[i].bracket, expected[i].bracket ? expected[i].bracket : modes.Bracket.None, 'bracket mismatch');
 	}
 };
 

src/vs/editor/test/common/modesTestUtils.ts

@@ -48,7 +48,7 @@ export function createLineContextFromTokenText(tokens: TokenText[]): modes.ILine
 
 	var indexSoFar = 0;
 	for (var i = 0; i < tokens.length; ++i){
-		processedTokens.push({ startIndex: indexSoFar, type: tokens[i].type, bracket: (tokens[i].bracket ? tokens[i].bracket : modes.Bracket.None) });
+		processedTokens.push({ startIndex: indexSoFar, type: tokens[i].type });
 		line += tokens[i].text;
 		indexSoFar += tokens[i].text.length;
 	}
@@ -95,10 +95,6 @@ class TestLineContext implements modes.ILineContext {
 		return this._tokens[tokenIndex].type;
 	}
 
-	public getTokenBracket(tokenIndex:number): modes.Bracket {
-		return this._tokens[tokenIndex].bracket;
-	}
-
 	public findIndexOfOffset(offset:number): number {
 		return Arrays.findIndexInSegmentsArray(this._tokens, offset);
 	}

src/vs/editor/test/common/modesUtil.ts

@@ -72,7 +72,7 @@ export function assertTokenization(tokenizationSupport: modes.ITokenizationSuppo
 		assert.ok(true, tests[i].line);
 		var result = tokenizationSupport.tokenize(tests[i].line, state);
 		if (tests[i].tokens) {
-			assert.deepEqual(generateRelaxedTokens(result.tokens, tests[i].tokens), tests[i].tokens, JSON.stringify(result.tokens, null, '\t'));
+			assert.deepEqual(toRelaxedTokens(result.tokens), toRelaxedTokens(tests[i].tokens), JSON.stringify(result.tokens, null, '\t'));
 		}
 
 		state = result.endState;
@@ -151,24 +151,13 @@ export function executeMonarchTokenizationTests(name:string, language:monarchTyp
 	executeTests(tokenizationSupport, tests);
 }
 
-function generateRelaxedTokens(actualTokens: modes.IToken[], expectedTokens: IRelaxedToken[]): IRelaxedToken[] {
-	var r = actualTokens.map((token, index) => {
-		// Remove bracket if it's missing in expectedTokens too
-		if (expectedTokens[index] && typeof expectedTokens[index].bracket !== 'undefined') {
-			return {
-				startIndex: token.startIndex,
-				type: token.type,
-				bracket: token.bracket
-			};
-		} else {
-			return {
-				startIndex: token.startIndex,
-				type: token.type
-			};
-		}
+function toRelaxedTokens(tokens: modes.IToken[]): IRelaxedToken[] {
+	return tokens.map((t) => {
+		return {
+			startIndex: t.startIndex,
+			type: t.type
+		};
 	});
-
-	return r;
 }
 
 function executeTest(tokenizationSupport: modes.ITokenizationSupport, tests:ITestItem[]): void {
@@ -187,5 +176,5 @@ function executeTest(tokenizationSupport: modes.ITokenizationSupport, tests:ITes
 }
 
 function assertTokens(actual:modes.IToken[], expected:IRelaxedToken[], message?:string): void {
-	assert.deepEqual(generateRelaxedTokens(actual, expected), expected, message + ': ' + JSON.stringify(actual, null, '\t'));
+	assert.deepEqual(toRelaxedTokens(actual), toRelaxedTokens(expected), message + ': ' + JSON.stringify(actual, null, '\t'));
 }
\ No newline at end of file

src/vs/languages/json/common/features/tokenization.ts

@@ -181,8 +181,7 @@ function tokenize(mode:Modes.IMode, comments:boolean, line:string, state:JSONSta
 		ret.endState = new JSONState(state.getMode(), state.getStateData(), scanner.getTokenError(), lastWasColon);
 		ret.tokens.push({
 			startIndex: offset,
-			type: type,
-			bracket: bracket
+			type: type
 		});
 	}
 

src/vs/languages/typescript/common/features/tokenization.ts

@@ -99,7 +99,7 @@ function tokenize(bracketTypeTable: { [i: number]: string }, tokenTypeTable: { [
 
 	function appendFn(startIndex:number, type:string, bracket:Modes.Bracket):void {
 		if(ret.tokens.length === 0 || bracket !== void 0 || arrays.tail(ret.tokens).type !== type) {
-			ret.tokens.push(new supports.Token(startIndex, type, bracket || Modes.Bracket.None));
+			ret.tokens.push(new supports.Token(startIndex, type));
 		}
 	}
 

src/vs/languages/typescript/test/common/tokenization.test.ts

@@ -4,8 +4,8 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
+import 'vs/languages/javascript/common/javascript.contribution';
 
-import javascriptMode = require('vs/languages/javascript/common/javascript');
 import EditorCommon = require('vs/editor/common/editorCommon');
 import Modes = require('vs/editor/common/modes');
 import modesUtil = require('vs/editor/test/common/modesUtil');