Simplify tokenTree

src/vs/editor/contrib/smartSelect/common/tokenTree.ts

@@ -6,13 +6,12 @@
 
 import {Position} from 'vs/editor/common/core/position';
 import {Range} from 'vs/editor/common/core/range';
-import {IModel, IPosition, IRichEditBracket} from 'vs/editor/common/editorCommon';
-import {LineTokens} from 'vs/editor/common/core/lineTokens';
+import {IModel, IPosition} from 'vs/editor/common/editorCommon';
+import {LineToken} from 'vs/editor/common/core/lineTokens';
 import {IRichEditBrackets} from 'vs/editor/common/modes';
 import {ignoreBracketsInToken} from 'vs/editor/common/modes/supports';
 import {BracketsUtils} from 'vs/editor/common/modes/supports/richEditBrackets';
 import {LanguageConfigurationRegistry} from 'vs/editor/common/modes/languageConfigurationRegistry';
-import {ModeTransition} from 'vs/editor/common/core/modeTransition';
 
 export const enum TokenTreeBracket {
 	None = 0,
@@ -102,11 +101,18 @@ export class Block extends Node {
 	}
 }
 
-interface Token {
+class Token {
+	_tokenBrand: void;
+
 	range: Range;
-	bracket: TokenTreeBracket;
 	type: string;
-	__debugContent?: string;
+	bracket: TokenTreeBracket;
+
+	constructor(range:Range, type: string, bracket: TokenTreeBracket) {
+		this.range = range;
+		this.type = type;
+		this.bracket = bracket;
+	}
 }
 
 function newNode(token: Token): Node {
@@ -116,128 +122,154 @@ function newNode(token: Token): Node {
 	return node;
 }
 
-class TokenScanner {
+class RawToken {
+	_basicTokenBrand: void;
+
+	public lineNumber: number;
+	public lineText: string;
+	public startOffset: number;
+	public endOffset: number;
+	public type: string;
+	public modeId: string;
+
+	constructor(source:LineToken, lineNumber:number, lineText:string) {
+		this.lineNumber = lineNumber;
+		this.lineText = lineText;
+		this.startOffset = source.startOffset;
+		this.endOffset = source.endOffset;
+		this.type = source.type;
+		this.modeId = source.modeId;
+	}
+}
+
+class ModelRawTokenScanner {
 
 	private _model: IModel;
+	private _lineCount: number;
 	private _versionId: number;
-	private _currentLineNumber: number;
-	private _currentTokenIndex: number;
-	private _currentTokenStart: number;
-	private _currentLineTokens: LineTokens;
-	private _currentLineModeTransitions: ModeTransition[];
-	private _currentModeIndex: number;
-	private _nextModeStart: number;
-	private _currentModeBrackets: IRichEditBrackets;
-	private _currentLineText: string;
+	private _lineNumber: number;
+	private _lineText: string;
+	private _next: LineToken;
 
-	constructor(model: IModel) {
+	constructor(model:IModel) {
 		this._model = model;
-		this._versionId = model.getVersionId();
-		this._currentLineNumber = 1;
+		this._lineCount = this._model.getLineCount();
+		this._versionId = this._model.getVersionId();
+		this._lineNumber = 0;
+		this._lineText = null;
+		this._advance();
 	}
 
-	next(): Token {
-		if (this._versionId !== this._model.getVersionId()) {
-			// model has been modified
-			return null;
+	private _advance(): void {
+		this._next = (this._next ? this._next.next() : null);
+		while (!this._next && this._lineNumber < this._lineCount) {
+			this._lineNumber++;
+			this._lineText = this._model.getLineContent(this._lineNumber);
+			let currentLineTokens = this._model.getLineTokens(this._lineNumber);
+			this._next = currentLineTokens.firstToken();
 		}
-		if (this._currentLineNumber >= this._model.getLineCount() + 1) {
-			// all line visisted
+	}
+
+	public next(): RawToken {
+		if (!this._next) {
 			return null;
 		}
-		if (!this._currentLineTokens) {
-			// no tokens for this line
-			this._currentLineTokens = this._model.getLineTokens(this._currentLineNumber);
-			this._currentLineText = this._model.getLineContent(this._currentLineNumber);
-			this._currentLineModeTransitions = this._model._getLineModeTransitions(this._currentLineNumber);
-			this._currentTokenIndex = 0;
-			this._currentTokenStart = 0;
-			this._currentModeIndex = -1;
-			this._nextModeStart = 0;
-		}
-		if (this._currentTokenIndex >= this._currentLineTokens.getTokenCount()) {
-			// last token of line visited
-			this._currentLineNumber += 1;
-			this._currentLineTokens = null;
-			return this.next();
+		if (this._model.getVersionId() !== this._versionId) {
+			return null;
 		}
 
-		if (this._currentTokenStart >= this._nextModeStart) {
-			this._currentModeIndex++;
-			this._nextModeStart = (this._currentModeIndex + 1 < this._currentLineModeTransitions.length ? this._currentLineModeTransitions[this._currentModeIndex + 1].startIndex : this._currentLineText.length + 1);
-			let mode = (this._currentModeIndex < this._currentLineModeTransitions.length ? this._currentLineModeTransitions[this._currentModeIndex] : null);
-			this._currentModeBrackets = (mode ? LanguageConfigurationRegistry.getBracketsSupport(mode.modeId) : null);
-		}
+		let result = new RawToken(this._next, this._lineNumber, this._lineText);
+		this._advance();
+		return result;
+	}
+}
 
-		let tokenType = this._currentLineTokens.getTokenType(this._currentTokenIndex);
-		let tokenEndIndex = this._currentLineTokens.getTokenEndOffset(this._currentTokenIndex);
-		let tmpTokenEndIndex = tokenEndIndex;
+class TokenScanner {
 
-		let nextBracket: Range = null;
-		if (this._currentModeBrackets && !ignoreBracketsInToken(tokenType)) {
-			nextBracket = BracketsUtils.findNextBracketInToken(this._currentModeBrackets.forwardRegex, this._currentLineNumber, this._currentLineText, this._currentTokenStart, tokenEndIndex);
-		}
+	private _rawTokenScanner: ModelRawTokenScanner;
+	private _nextBuff: Token[];
 
-		if (nextBracket && this._currentTokenStart < nextBracket.startColumn - 1) {
-			// found a bracket, but it is not at the beginning of the token
-			tmpTokenEndIndex = nextBracket.startColumn - 1;
-			nextBracket = null;
-		}
+	private _cachedModeBrackets: IRichEditBrackets;
+	private _cachedModeId: string;
+
+	constructor(model: IModel) {
+		this._rawTokenScanner = new ModelRawTokenScanner(model);
+		this._nextBuff = [];
+		this._cachedModeBrackets = null;
+		this._cachedModeId = null;
+	}
 
-		let bracketData: IRichEditBracket = null;
-		let bracketIsOpen: boolean = false;
-		if (nextBracket) {
-			let bracketText = this._currentLineText.substring(nextBracket.startColumn - 1, nextBracket.endColumn - 1);
-			bracketText = bracketText.toLowerCase();
+	next(): Token {
+		if (this._nextBuff.length > 0) {
+			return this._nextBuff.shift();
+		}
 
-			bracketData = this._currentModeBrackets.textIsBracket[bracketText];
-			bracketIsOpen = this._currentModeBrackets.textIsOpenBracket[bracketText];
+		const token = this._rawTokenScanner.next();
+		if (!token) {
+			return null;
+		}
+		const lineNumber = token.lineNumber;
+		const lineText = token.lineText;
+		const tokenType = token.type;
+		let startOffset = token.startOffset;
+		const endOffset = token.endOffset;
+
+		if (this._cachedModeId !== token.modeId) {
+			this._cachedModeId = token.modeId;
+			this._cachedModeBrackets = LanguageConfigurationRegistry.getBracketsSupport(this._cachedModeId);
+		}
+		const modeBrackets = this._cachedModeBrackets;
+
+		if (!modeBrackets || ignoreBracketsInToken(tokenType)) {
+			return new Token(
+				new Range(lineNumber, startOffset + 1, lineNumber, endOffset + 1),
+				tokenType,
+				TokenTreeBracket.None
+			);
 		}
 
-		if (!bracketData) {
-			let token: Token = {
-				type: tokenType,
-				bracket: TokenTreeBracket.None,
-				range: new Range(
-					this._currentLineNumber,
-					1 + this._currentTokenStart,
-					this._currentLineNumber,
-					1 + tmpTokenEndIndex
-				)
-			};
-			// console.log('TOKEN: <<' + this._currentLineText.substring(this._currentTokenStart, tmpTokenEndIndex) + '>>');
-
-			if (tmpTokenEndIndex < tokenEndIndex) {
-				// there is a bracket somewhere in this token...
-				this._currentTokenStart = tmpTokenEndIndex;
-			} else {
-				this._currentTokenIndex += 1;
-				this._currentTokenStart = (this._currentTokenIndex < this._currentLineTokens.getTokenCount() ? this._currentLineTokens.getTokenStartOffset(this._currentTokenIndex) : 0);
+		let foundBracket: Range;
+		do {
+			foundBracket = BracketsUtils.findNextBracketInToken(modeBrackets.forwardRegex, lineNumber, lineText, startOffset, endOffset);
+			if (foundBracket) {
+				const foundBracketStartOffset = foundBracket.startColumn - 1;
+				const foundBracketEndOffset = foundBracket.endColumn - 1;
+
+				if (startOffset < foundBracketStartOffset) {
+					// there is some text before this bracket in this token
+					this._nextBuff.push(new Token(
+						new Range(lineNumber, startOffset + 1, lineNumber, foundBracketStartOffset + 1),
+						tokenType,
+						TokenTreeBracket.None
+					));
+				}
+
+				let bracketText = lineText.substring(foundBracketStartOffset, foundBracketEndOffset);
+				bracketText = bracketText.toLowerCase();
+
+				const bracketData = modeBrackets.textIsBracket[bracketText];
+				const bracketIsOpen = modeBrackets.textIsOpenBracket[bracketText];
+
+				this._nextBuff.push(new Token(
+					new Range(lineNumber, foundBracketStartOffset + 1, lineNumber, foundBracketEndOffset + 1),
+					`${bracketData.modeId};${bracketData.open};${bracketData.close}`,
+					bracketIsOpen ? TokenTreeBracket.Open : TokenTreeBracket.Close
+				));
+
+				startOffset = foundBracketEndOffset;
 			}
-			return token;
+		} while(foundBracket);
+
+		if (startOffset < endOffset) {
+			// there is some remaining none-bracket text in this token
+			this._nextBuff.push(new Token(
+				new Range(lineNumber, startOffset + 1, lineNumber, endOffset + 1),
+				tokenType,
+				TokenTreeBracket.None
+			));
 		}
 
-		let type = `${bracketData.modeId};${bracketData.open};${bracketData.close}`;
-		let token: Token = {
-			type: type,
-			bracket: bracketIsOpen ? TokenTreeBracket.Open : TokenTreeBracket.Close,
-			range: new Range(
-				this._currentLineNumber,
-				1 + this._currentTokenStart,
-				this._currentLineNumber,
-				nextBracket.endColumn
-			)
-		};
-		// console.log('BRACKET: <<' + this._currentLineText.substring(this._currentTokenStart, nextBracket.endColumn - 1) + '>>');
-
-		if (nextBracket.endColumn - 1 < tokenEndIndex) {
-			// found a bracket, but it is not at the end of the token
-			this._currentTokenStart = nextBracket.endColumn - 1;
-		} else {
-			this._currentTokenIndex += 1;
-			this._currentTokenStart = (this._currentTokenIndex < this._currentLineTokens.getTokenCount() ? this._currentLineTokens.getTokenStartOffset(this._currentTokenIndex) : 0);
-		}
-		return token;
+		return this._nextBuff.shift();
 	}
 }