Do not generate brackets when tokenizing JS/TS

src/vs/languages/less/test/common/colorizer.test.ts

@@ -631,7 +631,6 @@ suite('LESS-tokenization', () => {
 				{ startIndex:16, type: '' },
 				{ startIndex:17, type: 'identifier.js' },
 				{ startIndex:24, type: 'delimiter.parenthesis.js' },
-				{ startIndex:25, type: 'delimiter.parenthesis.js' },
 				{ startIndex:26, type: 'punctuation.backtick.less' },
 				{ startIndex:27, type: '' },
 				{ startIndex:28, type: 'comment.less' }

src/vs/languages/typescript/common/features/tokenization.ts

@@ -95,8 +95,8 @@ function tokenize(bracketTypeTable: { [i: number]: string }, tokenTypeTable: { [
 		modeTransitions: [{ startIndex: offsetDelta, mode: state.getMode() }],
 	};
 
-	function appendFn(startIndex:number, type:string, bracket:Modes.Bracket):void {
-		if(ret.tokens.length === 0 || bracket !== void 0 || arrays.tail(ret.tokens).type !== type) {
+	function appendFn(startIndex:number, type:string):void {
+		if(ret.tokens.length === 0 || arrays.tail(ret.tokens).type !== type) {
 			ret.tokens.push(new supports.Token(startIndex, type));
 		}
 	}
@@ -116,28 +116,25 @@ function tokenize(bracketTypeTable: { [i: number]: string }, tokenTypeTable: { [
 
 	for (let entry of result.entries) {
 
-		var bracket = Modes.Bracket.None,
-			type: string;
+		var type: string;
 
 		if (entry.classification === ts.TokenClass.Punctuation) {
 			// punctions: check for brackets: (){}[]
 			var ch = text.charCodeAt(offset);
-			bracket = characterToBracket[ch] || Modes.Bracket.None;
 			type = bracketTypeTable[ch] || tokenTypeTable[entry.classification];
-			appendFn(offset + offsetDelta, type, bracket);
+			appendFn(offset + offsetDelta, type);
 
 		} else if (entry.classification === ts.TokenClass.Comment) {
 			// comments: check for JSDoc, block, and line comments
 			if (ret.endState.inJsDocComment || /\/\*\*.*\*\//.test(text.substr(offset, entry.length))) {
-				appendFn(offset + offsetDelta, isTypeScript ? 'comment.doc.ts' : 'comment.doc.js', Modes.Bracket.None);
+				appendFn(offset + offsetDelta, isTypeScript ? 'comment.doc.ts' : 'comment.doc.js');
 			} else {
-				appendFn(offset + offsetDelta, isTypeScript ? 'comment.ts' : 'comment.js', Modes.Bracket.None);
+				appendFn(offset + offsetDelta, isTypeScript ? 'comment.ts' : 'comment.js');
 			}
 		} else {
 			// everything else
 			appendFn(offset + offsetDelta,
-				tokenTypeTable[entry.classification] || strings.empty,
-				void 0);
+				tokenTypeTable[entry.classification] || strings.empty);
 		}
 
 		offset += entry.length;
@@ -146,14 +143,6 @@ function tokenize(bracketTypeTable: { [i: number]: string }, tokenTypeTable: { [
 	return ret;
 }
 
-var characterToBracket = collections.createNumberDictionary<number>();
-characterToBracket['('.charCodeAt(0)] = Modes.Bracket.Open;
-characterToBracket[')'.charCodeAt(0)] = Modes.Bracket.Close;
-characterToBracket['{'.charCodeAt(0)] = Modes.Bracket.Open;
-characterToBracket['}'.charCodeAt(0)] = Modes.Bracket.Close;
-characterToBracket['['.charCodeAt(0)] = Modes.Bracket.Open;
-characterToBracket[']'.charCodeAt(0)] = Modes.Bracket.Close;
-
 var tsBracketTypeTable = collections.createNumberDictionary<string>();
 tsBracketTypeTable['('.charCodeAt(0)] = 'delimiter.parenthesis.ts';
 tsBracketTypeTable[')'.charCodeAt(0)] = 'delimiter.parenthesis.ts';
@@ -189,9 +178,9 @@ jsTokenTypeTable[ts.TokenClass.RegExpLiteral] = 'regexp.js';
 jsTokenTypeTable[ts.TokenClass.StringLiteral] = 'string.js';
 
 
-function checkSheBang(state: State, deltaOffset: number, line: string, appendFn: (startIndex: number, type: string, bracket: Modes.Bracket) => void): boolean {
+function checkSheBang(state: State, deltaOffset: number, line: string, appendFn: (startIndex: number, type: string) => void): boolean {
 	if (line.indexOf('#!') === 0) {
-		appendFn(deltaOffset, 'comment.shebang', Modes.Bracket.None);
+		appendFn(deltaOffset, 'comment.shebang');
 		return true;
 	}
 }

src/vs/languages/typescript/test/common/tokenization.test.ts

@@ -6,7 +6,6 @@
 
 import 'vs/languages/javascript/common/javascript.contribution';
 
-import EditorCommon = require('vs/editor/common/editorCommon');
 import Modes = require('vs/editor/common/modes');
 import modesUtil = require('vs/editor/test/common/modesUtil');
 
@@ -68,12 +67,11 @@ suite('TS/JS - syntax highlighting', () => {
 				{ startIndex: 6, type: 'delimiter.js' },
 				{ startIndex: 7, type: '' },
 				{ startIndex: 8, type: 'keyword.js' },
-				{ startIndex: 16, type: 'delimiter.parenthesis.js', bracket: Modes.Bracket.Open },
-				{ startIndex: 17, type: 'delimiter.parenthesis.js', bracket: Modes.Bracket.Close },
+				{ startIndex: 16, type: 'delimiter.parenthesis.js' },
 				{ startIndex: 18, type: '' },
-				{ startIndex: 19, type: 'delimiter.bracket.js', bracket: Modes.Bracket.Open },
+				{ startIndex: 19, type: 'delimiter.bracket.js' },
 				{ startIndex: 20, type: '' },
-				{ startIndex: 21, type: 'delimiter.bracket.js', bracket: Modes.Bracket.Close },
+				{ startIndex: 21, type: 'delimiter.bracket.js' },
 				{ startIndex: 22, type: 'delimiter.js' }
 			]}],
 
@@ -457,58 +455,54 @@ suite('TS/JS - syntax highlighting', () => {
 				{ startIndex: 11, type: 'delimiter.js' }
 			]}],
 
-			// Bracket Matching
 			[{
 			line: '{ key: 123 }',
 			tokens: [
-				{ startIndex: 0, type: 'delimiter.bracket.js', bracket: Modes.Bracket.Open },
+				{ startIndex: 0, type: 'delimiter.bracket.js' },
 				{ startIndex: 1, type: '' },
 				{ startIndex: 2, type: 'identifier.js' },
 				{ startIndex: 5, type: 'delimiter.js' },
 				{ startIndex: 6, type: '' },
 				{ startIndex: 7, type: 'number.js' },
 				{ startIndex: 10, type: '' },
-				{ startIndex: 11, type: 'delimiter.bracket.js', bracket: Modes.Bracket.Close }
+				{ startIndex: 11, type: 'delimiter.bracket.js' }
 			]}],
 
 			[{
 			line: '[1,2,3]',
 			tokens: [
-				{ startIndex: 0, type: 'delimiter.array.js', bracket: Modes.Bracket.Open },
+				{ startIndex: 0, type: 'delimiter.array.js' },
 				{ startIndex: 1, type: 'number.js' },
 				{ startIndex: 2, type: 'delimiter.js' },
 				{ startIndex: 3, type: 'number.js' },
 				{ startIndex: 4, type: 'delimiter.js' },
 				{ startIndex: 5, type: 'number.js' },
-				{ startIndex: 6, type: 'delimiter.array.js', bracket: Modes.Bracket.Close }
+				{ startIndex: 6, type: 'delimiter.array.js' }
 			]}],
 
 			[{
 			line: 'foo(123);',
 			tokens: [
 				{ startIndex: 0, type: 'identifier.js' },
-				{ startIndex: 3, type: 'delimiter.parenthesis.js', bracket: Modes.Bracket.Open },
+				{ startIndex: 3, type: 'delimiter.parenthesis.js' },
 				{ startIndex: 4, type: 'number.js' },
-				{ startIndex: 7, type: 'delimiter.parenthesis.js', bracket: Modes.Bracket.Close },
+				{ startIndex: 7, type: 'delimiter.parenthesis.js' },
 				{ startIndex: 8, type: 'delimiter.js' }
 			]}],
 
 			[{
 			line: '{a:{b:[]}}',
 			tokens: [
-				{ startIndex: 0, type: 'delimiter.bracket.js', bracket: Modes.Bracket.Open },
+				{ startIndex: 0, type: 'delimiter.bracket.js' },
 				{ startIndex: 1, type: 'identifier.js' },
 				{ startIndex: 2, type: 'delimiter.js' },
-				{ startIndex: 3, type: 'delimiter.bracket.js', bracket: Modes.Bracket.Open },
+				{ startIndex: 3, type: 'delimiter.bracket.js' },
 				{ startIndex: 4, type: 'identifier.js' },
 				{ startIndex: 5, type: 'delimiter.js' },
-				{ startIndex: 6, type: 'delimiter.array.js', bracket: Modes.Bracket.Open },
-				{ startIndex: 7, type: 'delimiter.array.js', bracket: Modes.Bracket.Close },
-				{ startIndex: 8, type: 'delimiter.bracket.js', bracket: Modes.Bracket.Close },
-				{ startIndex: 9, type: 'delimiter.bracket.js', bracket: Modes.Bracket.Close }
+				{ startIndex: 6, type: 'delimiter.array.js' },
+				{ startIndex: 8, type: 'delimiter.bracket.js' }
 			]}],
 
-			// No Bracket Matching inside strings
 			[{
 			line: 'x = "[{()}]"',
 			tokens: [