Introduce and begin adopting StandardTokenType

src/vs/editor/common/core/lineTokens.ts

@@ -8,6 +8,33 @@ import { TokensBinaryEncoding, TokensInflatorMap } from 'vs/editor/common/model/
 import { ModeTransition } from 'vs/editor/common/core/modeTransition';
 import { ViewLineToken } from 'vs/editor/common/core/viewLineToken';
 
+/**
+ * A standard token type.
+ */
+export const enum StandardTokenType {
+	Other = 0,
+	Comment = 1,
+	String = 2,
+	RegEx = 3
+}
+
+const STANDARD_TOKEN_TYPE_REGEXP = /\b(comment|string|regex)\b/;
+function toStandardTokenType(tokenType: string): StandardTokenType {
+	let m = tokenType.match(STANDARD_TOKEN_TYPE_REGEXP);
+	if (!m) {
+		return StandardTokenType.Other;
+	}
+	switch (m[1]) {
+		case 'comment':
+			return StandardTokenType.Comment;
+		case 'string':
+			return StandardTokenType.String;
+		case 'regex':
+			return StandardTokenType.RegEx;
+	}
+	throw new Error('Unexpected match for standard token type!');
+}
+
 export class LineToken {
 	_lineTokenBrand: void;
 
@@ -18,6 +45,7 @@ export class LineToken {
 	public readonly startOffset: number;
 	public readonly endOffset: number;
 	public readonly type: string;
+	public readonly standardType: StandardTokenType;
 	public readonly modeId: string;
 	public readonly hasPrev: boolean;
 	public readonly hasNext: boolean;
@@ -30,6 +58,7 @@ export class LineToken {
 		this.startOffset = this._source.getTokenStartOffset(this._tokenIndex);
 		this.endOffset = this._source.getTokenEndOffset(this._tokenIndex);
 		this.type = this._source.getTokenType(this._tokenIndex);
+		this.standardType = toStandardTokenType(this.type);
 		this.modeId = this._source.modeTransitions[this._modeIndex].modeId;
 		this.hasPrev = (this._tokenIndex > 0);
 		this.hasNext = (this._tokenIndex + 1 < this._source.getTokenCount());
@@ -106,6 +135,10 @@ export class LineTokens {
 		return TokensBinaryEncoding.getType(this._map, this._tokens[tokenIndex]);
 	}
 
+	public getStandardTokenType(tokenIndex: number): StandardTokenType {
+		return toStandardTokenType(this.getTokenType(tokenIndex));
+	}
+
 	public getTokenEndOffset(tokenIndex: number): number {
 		if (tokenIndex + 1 < this._tokens.length) {
 			return TokensBinaryEncoding.getStartIndex(this._tokens[tokenIndex + 1]);

src/vs/editor/common/model/textModelWithTokens.ts

@@ -16,7 +16,7 @@ import { TokenIterator } from 'vs/editor/common/model/tokenIterator';
 import { ITokenizationSupport, ILineTokens, IMode, IState, TokenizationRegistry } from 'vs/editor/common/modes';
 import { NULL_MODE_ID, nullTokenize } from 'vs/editor/common/modes/nullMode';
 import { ignoreBracketsInToken } from 'vs/editor/common/modes/supports';
-import { BracketsUtils, RichEditBrackets, IRichEditBracket } from 'vs/editor/common/modes/supports/richEditBrackets';
+import { BracketsUtils, RichEditBrackets, RichEditBracket } from 'vs/editor/common/modes/supports/richEditBrackets';
 import { ModeTransition } from 'vs/editor/common/core/modeTransition';
 import { TokensInflatorMap } from 'vs/editor/common/model/tokensBinaryEncoding';
 import { Position } from 'vs/editor/common/core/position';
@@ -548,7 +548,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 		const currentModeBrackets = LanguageConfigurationRegistry.getBracketsSupport(currentToken.modeId);
 
 		// check that the token is not to be ignored
-		if (currentModeBrackets && !ignoreBracketsInToken(currentToken.type)) {
+		if (currentModeBrackets && !ignoreBracketsInToken(currentToken.standardType)) {
 			// limit search to not go before `maxBracketLength`
 			let searchStartOffset = Math.max(currentToken.startOffset, position.column - 1 - currentModeBrackets.maxBracketLength);
 			// limit search to not go after `maxBracketLength`
@@ -599,7 +599,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 			const prevModeBrackets = LanguageConfigurationRegistry.getBracketsSupport(prevToken.modeId);
 
 			// check that previous token is not to be ignored
-			if (prevModeBrackets && !ignoreBracketsInToken(prevToken.type)) {
+			if (prevModeBrackets && !ignoreBracketsInToken(prevToken.standardType)) {
 				// limit search in case previous token is very large, there's no need to go beyond `maxBracketLength`
 				const searchStartOffset = Math.max(prevToken.startOffset, position.column - 1 - prevModeBrackets.maxBracketLength);
 				const searchEndOffset = currentToken.startOffset;
@@ -623,7 +623,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 		return null;
 	}
 
-	private _matchFoundBracket(foundBracket: Range, data: IRichEditBracket, isOpen: boolean): [Range, Range] {
+	private _matchFoundBracket(foundBracket: Range, data: RichEditBracket, isOpen: boolean): [Range, Range] {
 		if (isOpen) {
 			let matched = this._findMatchingBracketDown(data, foundBracket.getEndPosition());
 			if (matched) {
@@ -639,7 +639,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 		return null;
 	}
 
-	private _findMatchingBracketUp(bracket: IRichEditBracket, position: Position): Range {
+	private _findMatchingBracketUp(bracket: RichEditBracket, position: Position): Range {
 		// console.log('_findMatchingBracketUp: ', 'bracket: ', JSON.stringify(bracket), 'startPosition: ', String(position));
 
 		const modeId = bracket.modeId;
@@ -663,7 +663,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 			}
 
 			while (currentToken) {
-				if (currentToken.modeId === modeId && !ignoreBracketsInToken(currentToken.type)) {
+				if (currentToken.modeId === modeId && !ignoreBracketsInToken(currentToken.standardType)) {
 
 					while (true) {
 						let r = BracketsUtils.findPrevBracketInToken(reversedBracketRegex, lineNumber, lineText, currentToken.startOffset, searchStopOffset);
@@ -698,7 +698,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 		return null;
 	}
 
-	private _findMatchingBracketDown(bracket: IRichEditBracket, position: Position): Range {
+	private _findMatchingBracketDown(bracket: RichEditBracket, position: Position): Range {
 		// console.log('_findMatchingBracketDown: ', 'bracket: ', JSON.stringify(bracket), 'startPosition: ', String(position));
 
 		const modeId = bracket.modeId;
@@ -722,7 +722,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 			}
 
 			while (currentToken) {
-				if (currentToken.modeId === modeId && !ignoreBracketsInToken(currentToken.type)) {
+				if (currentToken.modeId === modeId && !ignoreBracketsInToken(currentToken.standardType)) {
 					while (true) {
 						let r = BracketsUtils.findNextBracketInToken(bracketRegex, lineNumber, lineText, searchStartOffset, currentToken.endOffset);
 						if (!r) {
@@ -782,7 +782,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 					modeId = currentToken.modeId;
 					modeBrackets = LanguageConfigurationRegistry.getBracketsSupport(modeId);
 				}
-				if (modeBrackets && !ignoreBracketsInToken(currentToken.type)) {
+				if (modeBrackets && !ignoreBracketsInToken(currentToken.standardType)) {
 					let r = BracketsUtils.findPrevBracketInToken(modeBrackets.reversedRegex, lineNumber, lineText, currentToken.startOffset, searchStopOffset);
 					if (r) {
 						return this._toFoundBracket(modeBrackets, r);
@@ -825,7 +825,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 					modeId = currentToken.modeId;
 					modeBrackets = LanguageConfigurationRegistry.getBracketsSupport(modeId);
 				}
-				if (modeBrackets && !ignoreBracketsInToken(currentToken.type)) {
+				if (modeBrackets && !ignoreBracketsInToken(currentToken.standardType)) {
 					let r = BracketsUtils.findNextBracketInToken(modeBrackets.forwardRegex, lineNumber, lineText, searchStartOffset, currentToken.endOffset);
 					if (r) {
 						return this._toFoundBracket(modeBrackets, r);

src/vs/editor/common/modes/supports.ts

@@ -8,7 +8,7 @@ import { TPromise } from 'vs/base/common/winjs.base';
 import * as modes from 'vs/editor/common/modes';
 import { ModeTransition } from 'vs/editor/common/core/modeTransition';
 import { Token } from 'vs/editor/common/core/token';
-import { LineTokens } from 'vs/editor/common/core/lineTokens';
+import { LineTokens, StandardTokenType } from 'vs/editor/common/core/lineTokens';
 
 export class RawLineTokens implements modes.ILineTokens {
 	_lineTokensBrand: void;
@@ -99,9 +99,16 @@ export class ScopedLineTokens {
 	public getTokenType(tokenIndex: number): string {
 		return this._actual.getTokenType(tokenIndex + this._firstTokenIndex);
 	}
+
+	public getStandardTokenType(tokenIndex: number): StandardTokenType {
+		return this._actual.getStandardTokenType(tokenIndex + this._firstTokenIndex);
+	}
 }
 
-const IGNORE_IN_TOKENS = /\b(comment|string|regex)\b/;
-export function ignoreBracketsInToken(tokenType: string): boolean {
-	return IGNORE_IN_TOKENS.test(tokenType);
+export function ignoreBracketsInToken(standardTokenType: StandardTokenType): boolean {
+	return (
+		standardTokenType === StandardTokenType.Comment
+		|| standardTokenType === StandardTokenType.String
+		|| standardTokenType === StandardTokenType.RegEx
+	);
 }

src/vs/editor/common/modes/supports/electricCharacter.ts

@@ -104,7 +104,7 @@ export class BracketElectricCharacterSupport {
 			return null;
 		}
 
-		if (!ignoreBracketsInToken(context.getTokenType(tokenIndex))) {
+		if (!ignoreBracketsInToken(context.getStandardTokenType(tokenIndex))) {
 			let r = BracketsUtils.findPrevBracketInToken(reversedBracketRegex, 1, lineText, tokenStart, tokenEnd);
 			if (r) {
 				let text = lineText.substring(r.startColumn - 1, r.endColumn - 1);

src/vs/editor/common/modes/supports/richEditBrackets.ts

@@ -13,33 +13,43 @@ interface ISimpleInternalBracket {
 	close: string;
 }
 
-export interface IRichEditBracket {
+export class RichEditBracket {
+	_richEditBracketBrand: void;
+
 	readonly modeId: string;
 	readonly open: string;
 	readonly close: string;
 	readonly forwardRegex: RegExp;
 	readonly reversedRegex: RegExp;
+
+	constructor(modeId: string, open: string, close: string, forwardRegex: RegExp, reversedRegex: RegExp) {
+		this.modeId = modeId;
+		this.open = open;
+		this.close = close;
+		this.forwardRegex = forwardRegex;
+		this.reversedRegex = reversedRegex;
+	}
 }
 
 export class RichEditBrackets {
 	_richEditBracketsBrand: void;
 
-	public readonly brackets: IRichEditBracket[];
+	public readonly brackets: RichEditBracket[];
 	public readonly forwardRegex: RegExp;
 	public readonly reversedRegex: RegExp;
 	public readonly maxBracketLength: number;
-	public readonly textIsBracket: { [text: string]: IRichEditBracket; };
+	public readonly textIsBracket: { [text: string]: RichEditBracket; };
 	public readonly textIsOpenBracket: { [text: string]: boolean; };
 
 	constructor(modeId: string, brackets: CharacterPair[]) {
 		this.brackets = brackets.map((b) => {
-			return {
-				modeId: modeId,
-				open: b[0],
-				close: b[1],
-				forwardRegex: getRegexForBracketPair({ open: b[0], close: b[1] }),
-				reversedRegex: getReversedRegexForBracketPair({ open: b[0], close: b[1] })
-			};
+			return new RichEditBracket(
+				modeId,
+				b[0],
+				b[1],
+				getRegexForBracketPair({ open: b[0], close: b[1] }),
+				getReversedRegexForBracketPair({ open: b[0], close: b[1] })
+			);
 		});
 		this.forwardRegex = getRegexForBrackets(this.brackets);
 		this.reversedRegex = getReversedRegexForBrackets(this.brackets);

src/vs/editor/contrib/smartSelect/common/tokenTree.ts

@@ -7,7 +7,7 @@
 import { Position } from 'vs/editor/common/core/position';
 import { Range } from 'vs/editor/common/core/range';
 import { IModel, IPosition } from 'vs/editor/common/editorCommon';
-import { LineToken } from 'vs/editor/common/core/lineTokens';
+import { LineToken, StandardTokenType } from 'vs/editor/common/core/lineTokens';
 import { ignoreBracketsInToken } from 'vs/editor/common/modes/supports';
 import { BracketsUtils, RichEditBrackets } from 'vs/editor/common/modes/supports/richEditBrackets';
 import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
@@ -103,14 +103,14 @@ export class Block extends Node {
 class Token {
 	_tokenBrand: void;
 
-	range: Range;
-	type: string;
-	bracket: TokenTreeBracket;
+	readonly range: Range;
+	readonly bracket: TokenTreeBracket;
+	readonly bracketType: string;
 
-	constructor(range: Range, type: string, bracket: TokenTreeBracket) {
+	constructor(range: Range, bracket: TokenTreeBracket, bracketType: string) {
 		this.range = range;
-		this.type = type;
 		this.bracket = bracket;
+		this.bracketType = bracketType;
 	}
 }
 
@@ -128,7 +128,7 @@ class RawToken {
 	public lineText: string;
 	public startOffset: number;
 	public endOffset: number;
-	public type: string;
+	public standardType: StandardTokenType;
 	public modeId: string;
 
 	constructor(source: LineToken, lineNumber: number, lineText: string) {
@@ -136,7 +136,7 @@ class RawToken {
 		this.lineText = lineText;
 		this.startOffset = source.startOffset;
 		this.endOffset = source.endOffset;
-		this.type = source.type;
+		this.standardType = source.standardType;
 		this.modeId = source.modeId;
 	}
 }
@@ -209,7 +209,7 @@ class TokenScanner {
 		}
 		const lineNumber = token.lineNumber;
 		const lineText = token.lineText;
-		const tokenType = token.type;
+		const standardTokenType = token.standardType;
 		let startOffset = token.startOffset;
 		const endOffset = token.endOffset;
 
@@ -219,11 +219,11 @@ class TokenScanner {
 		}
 		const modeBrackets = this._cachedModeBrackets;
 
-		if (!modeBrackets || ignoreBracketsInToken(tokenType)) {
+		if (!modeBrackets || ignoreBracketsInToken(standardTokenType)) {
 			return new Token(
 				new Range(lineNumber, startOffset + 1, lineNumber, endOffset + 1),
-				tokenType,
-				TokenTreeBracket.None
+				TokenTreeBracket.None,
+				null
 			);
 		}
 
@@ -238,8 +238,8 @@ class TokenScanner {
 					// there is some text before this bracket in this token
 					this._nextBuff.push(new Token(
 						new Range(lineNumber, startOffset + 1, lineNumber, foundBracketStartOffset + 1),
-						tokenType,
-						TokenTreeBracket.None
+						TokenTreeBracket.None,
+						null
 					));
 				}
 
@@ -251,8 +251,8 @@ class TokenScanner {
 
 				this._nextBuff.push(new Token(
 					new Range(lineNumber, foundBracketStartOffset + 1, lineNumber, foundBracketEndOffset + 1),
-					`${bracketData.modeId};${bracketData.open};${bracketData.close}`,
-					bracketIsOpen ? TokenTreeBracket.Open : TokenTreeBracket.Close
+					bracketIsOpen ? TokenTreeBracket.Open : TokenTreeBracket.Close,
+					`${bracketData.modeId};${bracketData.open};${bracketData.close}`
 				));
 
 				startOffset = foundBracketEndOffset;
@@ -263,8 +263,8 @@ class TokenScanner {
 			// there is some remaining none-bracket text in this token
 			this._nextBuff.push(new Token(
 				new Range(lineNumber, startOffset + 1, lineNumber, endOffset + 1),
-				tokenType,
-				TokenTreeBracket.None
+				TokenTreeBracket.None,
+				null
 			));
 		}
 
@@ -353,7 +353,7 @@ class TokenTreeBuilder {
 			accepted: boolean;
 
 		accepted = this._accept(token => {
-			bracketType = token.type;
+			bracketType = token.bracketType;
 			return token.bracket === TokenTreeBracket.Open;
 		});
 		if (!accepted) {
@@ -366,7 +366,7 @@ class TokenTreeBuilder {
 			// inside brackets
 		}
 
-		if (!this._accept(token => token.bracket === TokenTreeBracket.Close && token.type === bracketType)) {
+		if (!this._accept(token => token.bracket === TokenTreeBracket.Close && token.bracketType === bracketType)) {
 			// missing closing bracket -> return just a node list
 			var nodelist = new NodeList();
 			nodelist.append(bracket.open);