Merge LineToken -> Token

src/vs/editor/common/core/token.ts

@@ -4,6 +4,8 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
+import {Arrays} from 'vs/editor/common/core/arrays';
+
 export class Token {
 	_tokenBrand: void;
 
@@ -11,11 +13,36 @@ export class Token {
 	public type:string;
 
 	constructor(startIndex:number, type:string) {
-		this.startIndex = startIndex;
+		this.startIndex = startIndex|0;// @perf
 		this.type = type;
 	}
 
 	public toString(): string {
 		return '(' + this.startIndex + ', ' + this.type + ')';
 	}
+
+	public equals(other:Token): boolean {
+		return (
+			this.startIndex === other.startIndex
+			&& this.type === other.type
+		);
+	}
+
+	public static findIndexInSegmentsArray(arr:Token[], desiredIndex: number): number {
+		return Arrays.findIndexInSegmentsArray(arr, desiredIndex);
+	}
+
+	public static equalsArray(a:Token[], b:Token[]): boolean {
+		let aLen = a.length;
+		let bLen = b.length;
+		if (aLen !== bLen) {
+			return false;
+		}
+		for (let i = 0; i < aLen; i++) {
+			if (!a[i].equals(b[i])) {
+				return false;
+			}
+		}
+		return true;
+	}
 }

src/vs/editor/common/model/lineToken.ts

-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the MIT License. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-'use strict';
-
-import {Arrays} from 'vs/editor/common/core/arrays';
-
-/**
- * A token on a line.
- */
-export class LineToken {
-	public _lineTokenBrand: void;
-
-	public startIndex:number;
-	public type:string;
-
-	constructor(startIndex:number, type:string) {
-		this.startIndex = startIndex|0;// @perf
-		this.type = type;
-	}
-
-	public equals(other:LineToken): boolean {
-		return (
-			this.startIndex === other.startIndex
-			&& this.type === other.type
-		);
-	}
-
-	public static findIndexInSegmentsArray(arr:LineToken[], desiredIndex: number): number {
-		return Arrays.findIndexInSegmentsArray(arr, desiredIndex);
-	}
-
-	public static equalsArray(a:LineToken[], b:LineToken[]): boolean {
-		let aLen = a.length;
-		let bLen = b.length;
-		if (aLen !== bLen) {
-			return false;
-		}
-		for (let i = 0; i < aLen; i++) {
-			if (!a[i].equals(b[i])) {
-				return false;
-			}
-		}
-		return true;
-	}
-}
\ No newline at end of file

src/vs/editor/common/model/modelLine.ts

@@ -9,7 +9,7 @@ import {ILineTokens, IReadOnlyLineMarker} from 'vs/editor/common/editorCommon';
 import {IState} from 'vs/editor/common/modes';
 import {TokensBinaryEncoding, TokensInflatorMap} from 'vs/editor/common/model/tokensBinaryEncoding';
 import {ModeTransition} from 'vs/editor/common/core/modeTransition';
-import {LineToken} from 'vs/editor/common/model/lineToken';
+import {Token} from 'vs/editor/common/core/token';
 import {ViewLineToken} from 'vs/editor/common/core/viewLineToken';
 
 const START_INDEX_MASK = TokensBinaryEncoding.START_INDEX_MASK;
@@ -190,7 +190,7 @@ export class ModelLine {
 
 	// --- BEGIN TOKENS
 
-	public setTokens(map: TokensInflatorMap, tokens: LineToken[], topLevelModeId:string, modeTransitions:ModeTransition[]): void {
+	public setTokens(map: TokensInflatorMap, tokens: Token[], topLevelModeId:string, modeTransitions:ModeTransition[]): void {
 		this._lineTokens = toLineTokensFromInflated(map, tokens, this._text.length);
 		this._modeTransitions = toModeTransitions(topLevelModeId, modeTransitions);
 	}
@@ -697,7 +697,7 @@ export class ModelLine {
 	}
 }
 
-function toLineTokensFromInflated(map:TokensInflatorMap, tokens:LineToken[], textLength:number): number[] {
+function toLineTokensFromInflated(map:TokensInflatorMap, tokens:Token[], textLength:number): number[] {
 	if (textLength === 0) {
 		return null;
 	}

src/vs/editor/common/model/textModelWithTokens.ts

@@ -22,7 +22,6 @@ import {NullMode, NullState, nullTokenize} from 'vs/editor/common/modes/nullMode
 import {ignoreBracketsInToken} from 'vs/editor/common/modes/supports';
 import {BracketsUtils} from 'vs/editor/common/modes/supports/richEditBrackets';
 import {ModeTransition} from 'vs/editor/common/core/modeTransition';
-import {LineToken} from 'vs/editor/common/model/lineToken';
 import {TokensInflatorMap} from 'vs/editor/common/model/tokensBinaryEncoding';
 import {Position} from 'vs/editor/common/core/position';
 import {LanguageConfigurationRegistry} from 'vs/editor/common/modes/languageConfigurationRegistry';
@@ -476,17 +475,6 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 		}
 	}
 
-	private static _toLineTokens(tokens:Token[]): LineToken[] {
-		if (!tokens || tokens.length === 0) {
-			return [];
-		}
-		let result:LineToken[] = [];
-		for (let i = 0, len = tokens.length; i < len; i++) {
-			result[i] = new LineToken(tokens[i].startIndex, tokens[i].type);
-		}
-		return result;
-	}
-
 	private _beginBackgroundTokenization(): void {
 		if (this._shouldAutoTokenize() && this._revalidateTokensTimeout === -1) {
 			this._revalidateTokensTimeout = setTimeout(() => {
@@ -638,7 +626,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 				// Make sure there is at least the transition to the top-most mode
 				r.modeTransitions.push(new ModeTransition(0, this.getModeId()));
 			}
-			this._lines[lineIndex].setTokens(this._tokensInflatorMap, TextModelWithTokens._toLineTokens(r.tokens), this.getModeId(), r.modeTransitions);
+			this._lines[lineIndex].setTokens(this._tokensInflatorMap, r.tokens, this.getModeId(), r.modeTransitions);
 
 			if (this._lines[lineIndex].isInvalid) {
 				this._lines[lineIndex].isInvalid = false;

src/vs/editor/common/model/tokensBinaryEncoding.ts

@@ -7,7 +7,7 @@
 import {onUnexpectedError} from 'vs/base/common/errors';
 import * as strings from 'vs/base/common/strings';
 import {ViewLineToken} from 'vs/editor/common/core/viewLineToken';
-import {LineToken} from 'vs/editor/common/model/lineToken';
+import {Token} from 'vs/editor/common/core/token';
 
 const START_INDEX_MASK = 0xffffffff;
 const TYPE_MASK = 0xffff;
@@ -40,7 +40,7 @@ export class TokensBinaryEncoding {
 	public static START_INDEX_OFFSET = START_INDEX_OFFSET;
 	public static TYPE_OFFSET = TYPE_OFFSET;
 
-	public static deflateArr(map:TokensInflatorMap, tokens:LineToken[]): number[] {
+	public static deflateArr(map:TokensInflatorMap, tokens:Token[]): number[] {
 		if (tokens.length === 0) {
 			return DEFLATED_TOKENS_EMPTY_TEXT;
 		}
@@ -52,7 +52,7 @@ export class TokensBinaryEncoding {
 			len:number,
 			deflatedToken:number,
 			deflated:number,
-			token:LineToken,
+			token:Token,
 			inflateMap = map._inflate,
 			deflateMap = map._deflate,
 			prevStartIndex:number = -1,