View line tokens track an `endIndex` instead of a `startIndex`

src/vs/editor/browser/standalone/colorizer.ts

@@ -10,7 +10,7 @@ import { IModel } from 'vs/editor/common/editorCommon';
 import { TokenizationRegistry, ITokenizationSupport } from 'vs/editor/common/modes';
 import { IModeService } from 'vs/editor/common/services/modeService';
 import { renderViewLine, RenderLineInput } from 'vs/editor/common/viewLayout/viewLineRenderer';
-import { ViewLineToken, ViewLineTokens } from 'vs/editor/common/core/viewLineToken';
+import { ViewLineToken } from 'vs/editor/common/core/viewLineToken';
 import { LineTokens } from 'vs/editor/common/core/lineTokens';
 import * as strings from 'vs/base/common/strings';
 import { IStandaloneColorService } from 'vs/editor/common/services/standaloneColorService';
@@ -97,7 +97,8 @@ export class Colorizer {
 	public static colorizeLine(line: string, tokens: ViewLineToken[], tabSize: number = 4): string {
 		let renderResult = renderViewLine(new RenderLineInput(
 			line,
-			new ViewLineTokens(tokens, 0, line.length),
+			0,
+			tokens,
 			[],
 			tabSize,
 			0,
@@ -128,7 +129,8 @@ function _fakeColorize(lines: string[], tabSize: number): string {
 
 		let renderResult = renderViewLine(new RenderLineInput(
 			line,
-			new ViewLineTokens([], 0, line.length),
+			0,
+			[new ViewLineToken(line.length, '')],
 			[],
 			tabSize,
 			0,
@@ -155,7 +157,8 @@ function _actualColorize(lines: string[], tabSize: number, tokenizationSupport:
 		let lineTokens = new LineTokens(colorMap, tokenizeResult.tokens, line);
 		let renderResult = renderViewLine(new RenderLineInput(
 			line,
-			new ViewLineTokens(lineTokens.inflate(), 0, line.length),
+			0,
+			lineTokens.inflate(),
 			[],
 			tabSize,
 			0,

src/vs/editor/browser/viewParts/lines/viewLine.ts

@@ -102,6 +102,7 @@ export class ViewLine implements IVisibleLineData {
 
 		let renderLineInput = new RenderLineInput(
 			model.getLineContent(lineNumber),
+			model.getLineMinColumn(lineNumber) - 1,
 			model.getLineTokens(lineNumber),
 			actualInlineDecorations,
 			model.getTabSize(),

src/vs/editor/browser/widget/diffEditorWidget.ts

@@ -23,7 +23,7 @@ import { Decoration } from 'vs/editor/common/viewLayout/viewLineParts';
 import { renderViewLine, RenderLineInput } from 'vs/editor/common/viewLayout/viewLineRenderer';
 import * as editorBrowser from 'vs/editor/browser/editorBrowser';
 import { CodeEditor } from 'vs/editor/browser/codeEditor';
-import { ViewLineToken, ViewLineTokens } from 'vs/editor/common/core/viewLineToken';
+import { ViewLineToken } from 'vs/editor/common/core/viewLineToken';
 import { Configuration } from 'vs/editor/browser/config/configuration';
 import { Position } from 'vs/editor/common/core/position';
 import { Selection } from 'vs/editor/common/core/selection';
@@ -1885,12 +1885,12 @@ class InlineViewZonesComputer extends ViewZonesComputer {
 	private renderOriginalLine(count: number, originalModel: editorCommon.IModel, config: editorCommon.InternalEditorOptions, tabSize: number, lineNumber: number, decorations: InlineDecoration[]): string[] {
 		let lineContent = originalModel.getLineContent(lineNumber);
 
-		let lineTokens = new ViewLineTokens([new ViewLineToken(0, '')], 0, lineContent.length);
 		let actualDecorations = Decoration.filter(decorations, lineNumber, 1, lineContent.length + 1);
 
 		let r = renderViewLine(new RenderLineInput(
 			lineContent,
-			lineTokens,
+			0,
+			[new ViewLineToken(lineContent.length, '')],
 			actualDecorations,
 			tabSize,
 			config.fontInfo.spaceWidth,

src/vs/editor/common/core/lineTokens.ts

@@ -122,6 +122,10 @@ export class LineTokens {
 		return this._text;
 	}
 
+	public getLineLength(): number {
+		return this._textLength;
+	}
+
 	public getTokenStartOffset(tokenIndex: number): number {
 		return this._tokens[(tokenIndex << 1)];
 	}
@@ -190,10 +194,10 @@ export class LineTokens {
 	}
 
 	public inflate(): ViewLineToken[] {
-		return TokenMetadata.inflateArr(this._tokens);
+		return TokenMetadata.inflateArr(this._tokens, this._textLength);
 	}
 
-	public sliceAndInflate(startOffset: number, endOffset: number, deltaStartIndex: number): ViewLineToken[] {
-		return TokenMetadata.sliceAndInflate(this._tokens, startOffset, endOffset, deltaStartIndex);
+	public sliceAndInflate(startOffset: number, endOffset: number, deltaOffset: number): ViewLineToken[] {
+		return TokenMetadata.sliceAndInflate(this._tokens, startOffset, endOffset, deltaOffset, this._textLength);
 	}
 }

src/vs/editor/common/core/viewLineToken.ts

@@ -4,7 +4,10 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
-export class ViewLineToken2 {
+/**
+ * A token on a line.
+ */
+export class ViewLineToken {
 	_viewLineTokenBrand: void;
 
 	/**
@@ -17,74 +20,25 @@ export class ViewLineToken2 {
 		this.endIndex = endIndex;
 		this.type = type;
 	}
-}
-
-/**
- * A token on a line.
- */
-export class ViewLineToken {
-	_oldViewLineTokenBrand: void;
-
-	public readonly startIndex: number;
-	public readonly type: string;
-
-	constructor(startIndex: number, type: string) {
-		this.startIndex = startIndex | 0;// @perf
-		this.type = type;
-	}
 
-	public equals(other: ViewLineToken): boolean {
+	private static _equals(a: ViewLineToken, b: ViewLineToken): boolean {
 		return (
-			this.startIndex === other.startIndex
-			&& this.type === other.type
+			a.endIndex === b.endIndex
+			&& a.type === b.type
 		);
 	}
 
-	public static equalsArray(a: ViewLineToken[], b: ViewLineToken[]): boolean {
-		let aLen = a.length;
-		let bLen = b.length;
+	public static equalsArr(a: ViewLineToken[], b: ViewLineToken[]): boolean {
+		const aLen = a.length;
+		const bLen = b.length;
 		if (aLen !== bLen) {
 			return false;
 		}
 		for (let i = 0; i < aLen; i++) {
-			if (!a[i].equals(b[i])) {
+			if (!this._equals(a[i], b[i])) {
 				return false;
 			}
 		}
 		return true;
 	}
 }
-
-export class ViewLineTokens {
-	_viewLineTokensBrand: void;
-
-	private _lineTokens: ViewLineToken[];
-	private _fauxIndentLength: number;
-	private _textLength: number;
-
-	constructor(lineTokens: ViewLineToken[], fauxIndentLength: number, textLength: number) {
-		this._lineTokens = lineTokens;
-		this._fauxIndentLength = fauxIndentLength | 0;
-		this._textLength = textLength | 0;
-	}
-
-	public getTokens(): ViewLineToken[] {
-		return this._lineTokens;
-	}
-
-	public getFauxIndentLength(): number {
-		return this._fauxIndentLength;
-	}
-
-	public getTextLength(): number {
-		return this._textLength;
-	}
-
-	public equals(other: ViewLineTokens): boolean {
-		return (
-			this._fauxIndentLength === other._fauxIndentLength
-			&& this._textLength === other._textLength
-			&& ViewLineToken.equalsArray(this._lineTokens, other._lineTokens)
-		);
-	}
-}

src/vs/editor/common/model/tokensBinaryEncoding.ts

@@ -71,36 +71,35 @@ export class TokenMetadata {
 		return className;
 	}
 
-	public static inflateArr(tokens: Uint32Array): ViewLineToken[] {
-		let tokenCount = (tokens.length >>> 1);
+	public static inflateArr(tokens: Uint32Array, lineLength: number): ViewLineToken[] {
 		let result: ViewLineToken[] = [];
 
-		for (let i = 0; i < tokenCount; i++) {
-			let startOffset = tokens[(i << 1)];
+		for (let i = 0, len = (tokens.length >>> 1); i < len; i++) {
+			let endOffset = (i + 1 < len ? tokens[((i + 1) << 1)] : lineLength);
 			let metadata = tokens[(i << 1) + 1];
 
-			result[i] = new ViewLineToken(startOffset, this._getClassNameFromMetadata(metadata));
+			result[i] = new ViewLineToken(endOffset, this._getClassNameFromMetadata(metadata));
 		}
+
 		return result;
 	}
 
-	public static sliceAndInflate(tokens: Uint32Array, startOffset: number, endOffset: number, deltaStartIndex: number): ViewLineToken[] {
+	public static sliceAndInflate(tokens: Uint32Array, startOffset: number, endOffset: number, deltaOffset: number, lineLength: number): ViewLineToken[] {
 		let tokenIndex = this.findIndexInSegmentsArray(tokens, startOffset);
 		let result: ViewLineToken[] = [], resultLen = 0;
 
-		result[resultLen++] = new ViewLineToken(0, this._getClassNameFromMetadata(tokens[(tokenIndex << 1) + 1]));
-
-		for (let i = tokenIndex + 1, len = (tokens.length >>> 1); i < len; i++) {
-			let originalStartOffset = tokens[(i << 1)];
+		for (let i = tokenIndex, len = (tokens.length >>> 1); i < len; i++) {
+			let tokenStartOffset = tokens[(i << 1)];
 
-			if (originalStartOffset >= endOffset) {
+			if (tokenStartOffset >= endOffset) {
 				break;
 			}
 
-			let newStartOffset = originalStartOffset - startOffset + deltaStartIndex;
+			let tokenEndOffset = (i + 1 < len ? tokens[((i + 1) << 1)] : lineLength);
+			let newEndOffset = tokenEndOffset - startOffset + deltaOffset;
 			let metadata = tokens[(i << 1) + 1];
 
-			result[resultLen++] = new ViewLineToken(newStartOffset, this._getClassNameFromMetadata(metadata));
+			result[resultLen++] = new ViewLineToken(newEndOffset, this._getClassNameFromMetadata(metadata));
 		}
 
 		return result;

src/vs/editor/common/modes/textToHtmlTokenizer.ts

@@ -41,16 +41,11 @@ function _tokenizeToString(text: string, tokenizationSupport: ITokenizationSuppo
 		let viewLineTokens = lineTokens.inflate();
 
 		let startOffset = 0;
-		let className = viewLineTokens[0].type;
-
-		for (let j = 1, lenJ = viewLineTokens.length; j < lenJ; j++) {
-			let viewLineToken = viewLineTokens[j];
-
-			result += `<span class="${className}">${strings.escape(line.substring(startOffset, viewLineToken.startIndex))}</span>`;
-			startOffset = viewLineToken.startIndex;
-			className = viewLineToken.type;
+		for (let j = 0, lenJ = viewLineTokens.length; j < lenJ; j++) {
+			const viewLineToken = viewLineTokens[j];
+			result += `<span class="${viewLineToken.type}">${strings.escape(line.substring(startOffset, viewLineToken.endIndex))}</span>`;
+			startOffset = viewLineToken.endIndex;
 		}
-		result += `<span class="${className}">${strings.escape(line.substring(startOffset))}</span>`;
 
 		currentState = tokenizationResult.endState;
 	}

src/vs/editor/common/viewLayout/viewLineRenderer.ts

@@ -4,51 +4,11 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
-import { ViewLineToken, ViewLineTokens } from 'vs/editor/common/core/viewLineToken';
+import { ViewLineToken } from 'vs/editor/common/core/viewLineToken';
 import { CharCode } from 'vs/base/common/charCode';
 import { Decoration, LineDecorationsNormalizer } from 'vs/editor/common/viewLayout/viewLineParts';
 import * as strings from 'vs/base/common/strings';
 
-
-
-class ViewLineToken2 {
-	_viewLineTokenBrand: void;
-
-	/**
-	 * last char index of this token (not inclusive).
-	 */
-	public readonly endIndex: number;
-	public readonly type: string;
-
-	constructor(endIndex: number, type: string) {
-		this.endIndex = endIndex;
-		this.type = type;
-	}
-}
-
-/**
- * TODO@Alex: transform please
- */
-function transformPlease(tokens: ViewLineToken[], len: number): ViewLineToken2[] {
-	console.log(`input len::: `, len);
-	console.log(`input here::: `, tokens);
-	let result: ViewLineToken2[] = [];
-	for (let tokenIndex = 0, tokensLen = tokens.length; tokenIndex < tokensLen; tokenIndex++) {
-		if (tokens[tokenIndex].startIndex > len) {
-			break;
-			// throw new Error('TODO!');
-		}
-		let nextTokenStartIndex = (
-			tokenIndex + 1 < tokensLen
-				? Math.min(len, tokens[tokenIndex + 1].startIndex)
-				: len
-		);
-		result[tokenIndex] = new ViewLineToken2(nextTokenStartIndex, tokens[tokenIndex].type);
-	}
-	console.log(`result here:::: `, result);
-	return result;
-}
-
 export const enum RenderWhitespace {
 	None = 0,
 	Boundary = 1,
@@ -58,7 +18,8 @@ export const enum RenderWhitespace {
 export class RenderLineInput {
 
 	public readonly lineContent: string;
-	public readonly lineTokens: ViewLineTokens;
+	public readonly fauxIndentLength: number;
+	public readonly lineTokens: ViewLineToken[];
 	public readonly lineDecorations: Decoration[];
 	public readonly tabSize: number;
 	public readonly spaceWidth: number;
@@ -68,7 +29,8 @@ export class RenderLineInput {
 
 	constructor(
 		lineContent: string,
-		lineTokens: ViewLineTokens,
+		fauxIndentLength: number,
+		lineTokens: ViewLineToken[],
 		lineDecorations: Decoration[],
 		tabSize: number,
 		spaceWidth: number,
@@ -77,6 +39,7 @@ export class RenderLineInput {
 		renderControlCharacters: boolean,
 	) {
 		this.lineContent = lineContent;
+		this.fauxIndentLength = fauxIndentLength;
 		this.lineTokens = lineTokens;
 		this.lineDecorations = lineDecorations;
 		this.tabSize = tabSize;
@@ -95,13 +58,14 @@ export class RenderLineInput {
 	public equals(other: RenderLineInput): boolean {
 		return (
 			this.lineContent === other.lineContent
+			&& this.fauxIndentLength === other.fauxIndentLength
 			&& this.tabSize === other.tabSize
 			&& this.spaceWidth === other.spaceWidth
 			&& this.stopRenderingLineAfter === other.stopRenderingLineAfter
 			&& this.renderWhitespace === other.renderWhitespace
 			&& this.renderControlCharacters === other.renderControlCharacters
 			&& Decoration.equalsArr(this.lineDecorations, other.lineDecorations)
-			&& this.lineTokens.equals(other.lineTokens)
+			&& ViewLineToken.equalsArr(this.lineTokens, other.lineTokens)
 		);
 	}
 }
@@ -246,7 +210,7 @@ class ResolvedRenderLineInput {
 		public readonly lineContent: string,
 		public readonly len: number,
 		public readonly isOverflowing: boolean,
-		public readonly tokens: ViewLineToken2[],
+		public readonly tokens: ViewLineToken[],
 		public readonly lineDecorations: Decoration[],
 		public readonly tabSize: number,
 		public readonly spaceWidth: number,
@@ -271,13 +235,10 @@ function resolveRenderLineInput(input: RenderLineInput): ResolvedRenderLineInput
 		len = lineContent.length;
 	}
 
-	let tokens: ViewLineToken2[];
+	let tokens = removeOverflowing(input.lineTokens, len);
 	if (input.renderWhitespace === RenderWhitespace.All || input.renderWhitespace === RenderWhitespace.Boundary) {
-		tokens = _applyRenderWhitespace(lineContent, len, transformPlease(input.lineTokens.getTokens(), len), input.lineTokens.getFauxIndentLength(), input.tabSize, input.renderWhitespace === RenderWhitespace.Boundary);
-	} else {
-		tokens = transformPlease(input.lineTokens.getTokens(), len);
+		tokens = _applyRenderWhitespace(lineContent, len, tokens, input.fauxIndentLength, input.tabSize, input.renderWhitespace === RenderWhitespace.Boundary);
 	}
-
 	if (input.lineDecorations.length > 0) {
 		tokens = _applyInlineDecorations(lineContent, len, tokens, input.lineDecorations);
 	}
@@ -295,15 +256,38 @@ function resolveRenderLineInput(input: RenderLineInput): ResolvedRenderLineInput
 	);
 }
 
-function _applyRenderWhitespace(lineContent: string, len: number, tokens: ViewLineToken2[], fauxIndentLength: number, tabSize: number, onlyBoundary: boolean): ViewLineToken2[] {
+function removeOverflowing(tokens: ViewLineToken[], len: number): ViewLineToken[] {
+	if (tokens.length === 0) {
+		return tokens;
+	}
+	if (tokens[tokens.length - 1].endIndex === len) {
+		return tokens;
+	}
+	let result: ViewLineToken[] = [];
+	for (let tokenIndex = 0, tokensLen = tokens.length; tokenIndex < tokensLen; tokenIndex++) {
+		const endIndex = tokens[tokenIndex].endIndex;
+		if (endIndex === len) {
+			result[tokenIndex] = tokens[tokenIndex];
+			break;
+		}
+		if (endIndex > len) {
+			result[tokenIndex] = new ViewLineToken(len, tokens[tokenIndex].type);
+			break;
+		}
+		result[tokenIndex] = tokens[tokenIndex];
+	}
+	return result;
+}
+
+function _applyRenderWhitespace(lineContent: string, len: number, tokens: ViewLineToken[], fauxIndentLength: number, tabSize: number, onlyBoundary: boolean): ViewLineToken[] {
 
-	let result: ViewLineToken2[] = [], resultLen = 0;
+	let result: ViewLineToken[] = [], resultLen = 0;
 	let tokenIndex = 0;
 	let tokenType = tokens[tokenIndex].type;
 	let tokenEndIndex = tokens[tokenIndex].endIndex;
 
 	if (fauxIndentLength > 0) {
-		result[resultLen++] = new ViewLineToken2(fauxIndentLength, '');
+		result[resultLen++] = new ViewLineToken(fauxIndentLength, '');
 	}
 
 	let firstNonWhitespaceIndex = strings.firstNonWhitespaceIndex(lineContent);
@@ -359,13 +343,13 @@ function _applyRenderWhitespace(lineContent: string, len: number, tokens: ViewLi
 			// was in whitespace token
 			if (!isInWhitespace || tmpIndent >= tabSize) {
 				// leaving whitespace token or entering a new indent
-				result[resultLen++] = new ViewLineToken2(charIndex, 'vs-whitespace');
+				result[resultLen++] = new ViewLineToken(charIndex, 'vs-whitespace');
 				tmpIndent = tmpIndent % tabSize;
 			}
 		} else {
 			// was in regular token
 			if (charIndex === tokenEndIndex || (isInWhitespace && charIndex > fauxIndentLength)) {
-				result[resultLen++] = new ViewLineToken2(charIndex, tokenType);
+				result[resultLen++] = new ViewLineToken(charIndex, tokenType);
 				tmpIndent = tmpIndent % tabSize;
 			}
 		}
@@ -387,22 +371,22 @@ function _applyRenderWhitespace(lineContent: string, len: number, tokens: ViewLi
 
 	if (wasInWhitespace) {
 		// was in whitespace token
-		result[resultLen++] = new ViewLineToken2(len, 'vs-whitespace');
+		result[resultLen++] = new ViewLineToken(len, 'vs-whitespace');
 	} else {
 		// was in regular token
-		result[resultLen++] = new ViewLineToken2(len, tokenType);
+		result[resultLen++] = new ViewLineToken(len, tokenType);
 	}
 
 	return result;
 }
 
-function _applyInlineDecorations(lineContent: string, len: number, tokens: ViewLineToken2[], _lineDecorations: Decoration[]): ViewLineToken2[] {
+function _applyInlineDecorations(lineContent: string, len: number, tokens: ViewLineToken[], _lineDecorations: Decoration[]): ViewLineToken[] {
 	_lineDecorations.sort(Decoration.compare);
 	const lineDecorations = LineDecorationsNormalizer.normalize(_lineDecorations);
 	const lineDecorationsLen = lineDecorations.length;
 
 	let lineDecorationIndex = 0;
-	let result: ViewLineToken2[] = [], resultLen = 0, lastResultEndIndex = 0;
+	let result: ViewLineToken[] = [], resultLen = 0, lastResultEndIndex = 0;
 	for (let tokenIndex = 0, len = tokens.length; tokenIndex < len; tokenIndex++) {
 		const token = tokens[tokenIndex];
 		const tokenEndIndex = token.endIndex;
@@ -413,12 +397,12 @@ function _applyInlineDecorations(lineContent: string, len: number, tokens: ViewL
 
 			if (lineDecoration.startOffset > lastResultEndIndex) {
 				lastResultEndIndex = lineDecoration.startOffset;
-				result[resultLen++] = new ViewLineToken2(lastResultEndIndex, tokenType);
+				result[resultLen++] = new ViewLineToken(lastResultEndIndex, tokenType);
 			}
 
 			if (lineDecoration.endOffset + 1 < tokenEndIndex) {
 				lastResultEndIndex = lineDecoration.endOffset + 1;
-				result[resultLen++] = new ViewLineToken2(lastResultEndIndex, tokenType + ' ' + lineDecoration.className);
+				result[resultLen++] = new ViewLineToken(lastResultEndIndex, tokenType + ' ' + lineDecoration.className);
 				lineDecorationIndex++;
 			} else {
 				break;
@@ -427,7 +411,7 @@ function _applyInlineDecorations(lineContent: string, len: number, tokens: ViewL
 
 		if (tokenEndIndex > lastResultEndIndex) {
 			lastResultEndIndex = tokenEndIndex;
-			result[resultLen++] = new ViewLineToken2(lastResultEndIndex, tokenType);
+			result[resultLen++] = new ViewLineToken(lastResultEndIndex, tokenType);
 		}
 	}
 

src/vs/editor/common/viewModel/filteredLineTokens.ts

-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the MIT License. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-'use strict';
-
-import { ViewLineTokens } from 'vs/editor/common/core/viewLineToken';
-import { LineTokens } from 'vs/editor/common/core/lineTokens';
-
-export class FilteredLineTokens {
-	/**
-	 * [startOffset; endOffset) (i.e. do not include endOffset)
-	 */
-	public static create(original: LineTokens, startOffset: number, endOffset: number, deltaStartIndex: number): ViewLineTokens {
-		let inflatedTokens = original.sliceAndInflate(startOffset, endOffset, deltaStartIndex);
-		return new ViewLineTokens(
-			inflatedTokens,
-			deltaStartIndex,
-			endOffset - startOffset + deltaStartIndex
-		);
-	}
-}
-
-export class IdentityFilteredLineTokens {
-
-	public static create(original: LineTokens, textLength: number): ViewLineTokens {
-		let inflatedTokens = original.inflate();
-		return new ViewLineTokens(
-			inflatedTokens,
-			0,
-			textLength
-		);
-	}
-}

src/vs/editor/common/viewModel/splitLinesCollection.ts

@@ -8,10 +8,9 @@ import { Position } from 'vs/editor/common/core/position';
 import { Range } from 'vs/editor/common/core/range';
 import * as editorCommon from 'vs/editor/common/editorCommon';
 import { LineTokens } from 'vs/editor/common/core/lineTokens';
-import { FilteredLineTokens, IdentityFilteredLineTokens } from 'vs/editor/common/viewModel/filteredLineTokens';
 import { PrefixSumComputer } from 'vs/editor/common/viewModel/prefixSumComputer';
 import { ILinesCollection } from 'vs/editor/common/viewModel/viewModelImpl';
-import { ViewLineTokens } from 'vs/editor/common/core/viewLineToken';
+import { ViewLineToken } from 'vs/editor/common/core/viewLineToken';
 
 export class OutputPosition {
 	_outputPositionBrand: void;
@@ -49,7 +48,7 @@ export interface ISplitLine {
 	getOutputLineContent(model: IModel, myLineNumber: number, outputLineIndex: number): string;
 	getOutputLineMinColumn(model: IModel, myLineNumber: number, outputLineIndex: number): number;
 	getOutputLineMaxColumn(model: IModel, myLineNumber: number, outputLineIndex: number): number;
-	getOutputLineTokens(model: IModel, myLineNumber: number, outputLineIndex: number): ViewLineTokens;
+	getOutputLineTokens(model: IModel, myLineNumber: number, outputLineIndex: number): ViewLineToken[];
 	getInputColumnOfOutputPosition(outputLineIndex: number, outputColumn: number): number;
 	getOutputPositionOfInputPosition(deltaLineNumber: number, inputColumn: number): Position;
 }
@@ -87,8 +86,9 @@ class VisibleIdentitySplitLine implements ISplitLine {
 		return model.getLineMaxColumn(myLineNumber);
 	}
 
-	public getOutputLineTokens(model: IModel, myLineNumber: number, outputLineIndex: number): ViewLineTokens {
-		return IdentityFilteredLineTokens.create(model.getLineTokens(myLineNumber, true), model.getLineMaxColumn(myLineNumber) - 1);
+	public getOutputLineTokens(model: IModel, myLineNumber: number, outputLineIndex: number): ViewLineToken[] {
+		let lineTokens = model.getLineTokens(myLineNumber, true);
+		return lineTokens.inflate();
 	}
 
 	public getInputColumnOfOutputPosition(outputLineIndex: number, outputColumn: number): number {
@@ -133,7 +133,7 @@ class InvisibleIdentitySplitLine implements ISplitLine {
 		throw new Error('Not supported');
 	}
 
-	public getOutputLineTokens(model: IModel, myLineNumber: number, outputLineIndex: number): ViewLineTokens {
+	public getOutputLineTokens(model: IModel, myLineNumber: number, outputLineIndex: number): ViewLineToken[] {
 		throw new Error('Not supported');
 	}
 
@@ -223,7 +223,7 @@ export class SplitLine implements ISplitLine {
 		return this.getOutputLineContent(model, myLineNumber, outputLineIndex).length + 1;
 	}
 
-	public getOutputLineTokens(model: IModel, myLineNumber: number, outputLineIndex: number): ViewLineTokens {
+	public getOutputLineTokens(model: IModel, myLineNumber: number, outputLineIndex: number): ViewLineToken[] {
 		if (!this._isVisible) {
 			throw new Error('Not supported');
 		}
@@ -233,7 +233,9 @@ export class SplitLine implements ISplitLine {
 		if (outputLineIndex > 0) {
 			deltaStartIndex = this.wrappedIndentLength;
 		}
-		return FilteredLineTokens.create(model.getLineTokens(myLineNumber, true), startOffset, endOffset, deltaStartIndex);
+
+		let lineTokens = model.getLineTokens(myLineNumber, true);
+		return lineTokens.sliceAndInflate(startOffset, endOffset, deltaStartIndex);
 	}
 
 	public getInputColumnOfOutputPosition(outputLineIndex: number, outputColumn: number): number {
@@ -691,7 +693,7 @@ export class SplitLinesCollection implements ILinesCollection {
 		return this.lines[lineIndex].getOutputLineMaxColumn(this.model, lineIndex + 1, remainder);
 	}
 
-	public getOutputLineTokens(outputLineNumber: number): ViewLineTokens {
+	public getOutputLineTokens(outputLineNumber: number): ViewLineToken[] {
 		this._ensureValidState();
 		outputLineNumber = this._toValidOutputLineNumber(outputLineNumber);
 		let r = this.prefixSumComputer.getIndexOf(outputLineNumber - 1);

src/vs/editor/common/viewModel/viewModel.ts

@@ -6,7 +6,7 @@
 
 import { IEventEmitter } from 'vs/base/common/eventEmitter';
 import { IModelDecoration, EndOfLinePreference, IPosition } from 'vs/editor/common/editorCommon';
-import { ViewLineTokens } from 'vs/editor/common/core/viewLineToken';
+import { ViewLineToken } from 'vs/editor/common/core/viewLineToken';
 import { Position } from 'vs/editor/common/core/position';
 import { Range } from 'vs/editor/common/core/range';
 import { Selection } from 'vs/editor/common/core/selection';
@@ -31,7 +31,7 @@ export interface IViewModel extends IEventEmitter {
 	getLineMaxColumn(lineNumber: number): number;
 	getLineFirstNonWhitespaceColumn(lineNumber: number): number;
 	getLineLastNonWhitespaceColumn(lineNumber: number): number;
-	getLineTokens(lineNumber: number): ViewLineTokens;
+	getLineTokens(lineNumber: number): ViewLineToken[];
 	getDecorationsViewportData(startLineNumber: number, endLineNumber: number): IDecorationsViewportData;
 	getLineRenderLineNumber(lineNumber: number): string;
 	/**

src/vs/editor/common/viewModel/viewModelImpl.ts

@@ -14,7 +14,7 @@ import * as editorCommon from 'vs/editor/common/editorCommon';
 import { ViewModelCursors } from 'vs/editor/common/viewModel/viewModelCursors';
 import { ViewModelDecorations } from 'vs/editor/common/viewModel/viewModelDecorations';
 import { ViewModelDecoration, IDecorationsViewportData, IViewModel } from 'vs/editor/common/viewModel/viewModel';
-import { ViewLineTokens } from 'vs/editor/common/core/viewLineToken';
+import { ViewLineToken } from 'vs/editor/common/core/viewLineToken';
 
 export interface ILinesCollection {
 	setTabSize(newTabSize: number, emit: (evenType: string, payload: any) => void): boolean;
@@ -30,7 +30,7 @@ export interface ILinesCollection {
 	getOutputIndentGuide(outputLineNumber: number): number;
 	getOutputLineMinColumn(outputLineNumber: number): number;
 	getOutputLineMaxColumn(outputLineNumber: number): number;
-	getOutputLineTokens(outputLineNumber: number): ViewLineTokens;
+	getOutputLineTokens(outputLineNumber: number): ViewLineToken[];
 	convertOutputPositionToInputPosition(viewLineNumber: number, viewColumn: number): Position;
 	convertInputPositionToOutputPosition(inputLineNumber: number, inputColumn: number): Position;
 	setHiddenAreas(ranges: editorCommon.IRange[], emit: (evenType: string, payload: any) => void): void;
@@ -445,7 +445,7 @@ export class ViewModel extends EventEmitter implements IViewModel {
 		return result + 2;
 	}
 
-	public getLineTokens(lineNumber: number): ViewLineTokens {
+	public getLineTokens(lineNumber: number): ViewLineToken[] {
 		return this.lines.getOutputLineTokens(lineNumber);
 	}
 

src/vs/editor/test/common/model/model.line.test.ts

@@ -9,15 +9,12 @@ import { LineTokens } from 'vs/editor/common/core/lineTokens';
 import { ModelLine, ILineEdit, LineMarker, MarkersTracker } from 'vs/editor/common/model/modelLine';
 import { MetadataConsts } from 'vs/editor/common/modes';
 import { Position } from 'vs/editor/common/core/position';
+import { TokenMetadata } from 'vs/editor/common/model/tokensBinaryEncoding';
 
-function assertLineTokens(actual: LineTokens, expected: TestToken[]): void {
-	let inflatedActual = actual.inflate();
-	assert.deepEqual(inflatedActual, expected.map((token) => {
-		return {
-			startIndex: token.startOffset,
-			type: 'mtk' + token.color
-		};
-	}), 'Line tokens are equal');
+function assertLineTokens(_actual: LineTokens, _expected: TestToken[]): void {
+	let expected = TokenMetadata.inflateArr(TestToken.toTokens(_expected), _actual.getLineLength());
+	let actual = _actual.inflate();
+	assert.deepEqual(actual, expected);
 }
 
 const NO_TAB_SIZE = 0;

src/vs/editor/test/common/model/textModelWithTokens.test.ts

@@ -293,18 +293,18 @@ suite('TextModelWithTokens regression tests', () => {
 
 		let model = Model.createFromString('A model with\ntwo lines');
 
-		assertViewLineTokens(model, 1, true, [new ViewLineToken(0, 'mtk1')]);
-		assertViewLineTokens(model, 2, true, [new ViewLineToken(0, 'mtk1')]);
+		assertViewLineTokens(model, 1, true, [new ViewLineToken(12, 'mtk1')]);
+		assertViewLineTokens(model, 2, true, [new ViewLineToken(9, 'mtk1')]);
 
 		model.setMode(languageIdentifier1);
 
-		assertViewLineTokens(model, 1, true, [new ViewLineToken(0, 'mtk11')]);
-		assertViewLineTokens(model, 2, true, [new ViewLineToken(0, 'mtk12')]);
+		assertViewLineTokens(model, 1, true, [new ViewLineToken(12, 'mtk11')]);
+		assertViewLineTokens(model, 2, true, [new ViewLineToken(9, 'mtk12')]);
 
 		model.setMode(languageIdentifier2);
 
-		assertViewLineTokens(model, 1, false, [new ViewLineToken(0, 'mtk1')]);
-		assertViewLineTokens(model, 2, false, [new ViewLineToken(0, 'mtk1')]);
+		assertViewLineTokens(model, 1, false, [new ViewLineToken(12, 'mtk1')]);
+		assertViewLineTokens(model, 2, false, [new ViewLineToken(9, 'mtk1')]);
 
 		model.dispose();
 		registration1.dispose();

src/vs/editor/test/common/viewLayout/viewLineParts.test.ts

@@ -8,7 +8,7 @@ import * as assert from 'assert';
 import { DecorationSegment, LineDecorationsNormalizer, Decoration } from 'vs/editor/common/viewLayout/viewLineParts';
 import { Range } from 'vs/editor/common/core/range';
 import { RenderLineInput, renderViewLine } from 'vs/editor/common/viewLayout/viewLineRenderer';
-import { ViewLineToken, ViewLineTokens } from 'vs/editor/common/core/viewLineToken';
+import { ViewLineToken } from 'vs/editor/common/core/viewLineToken';
 import { InlineDecoration } from 'vs/editor/common/viewModel/viewModel';
 
 suite('Editor ViewLayout - ViewLineParts', () => {
@@ -58,7 +58,8 @@ suite('Editor ViewLayout - ViewLineParts', () => {
 	function testCreateLineParts(lineContent: string, tokens: ViewLineToken[], fauxIndentLength: number, renderWhitespace: 'none' | 'boundary' | 'all', expected: string): void {
 		let actual = renderViewLine(new RenderLineInput(
 			lineContent,
-			new ViewLineTokens(tokens, fauxIndentLength, lineContent.length),
+			fauxIndentLength,
+			tokens,
 			[],
 			4,
 			10,
@@ -74,7 +75,7 @@ suite('Editor ViewLayout - ViewLineParts', () => {
 		testCreateLineParts(
 			'Hello world!',
 			[
-				new ViewLineToken(0, '')
+				new ViewLineToken(12, '')
 			],
 			0,
 			'none',
@@ -89,8 +90,8 @@ suite('Editor ViewLayout - ViewLineParts', () => {
 		testCreateLineParts(
 			'Hello world!',
 			[
-				new ViewLineToken(0, 'a'),
-				new ViewLineToken(6, 'b')
+				new ViewLineToken(6, 'a'),
+				new ViewLineToken(12, 'b')
 			],
 			0,
 			'none',
@@ -106,9 +107,9 @@ suite('Editor ViewLayout - ViewLineParts', () => {
 		testCreateLineParts(
 			'    Hello world!    ',
 			[
-				new ViewLineToken(0, ''),
-				new ViewLineToken(4, 'a'),
-				new ViewLineToken(6, 'b')
+				new ViewLineToken(4, ''),
+				new ViewLineToken(6, 'a'),
+				new ViewLineToken(20, 'b')
 			],
 			0,
 			'boundary',
@@ -126,9 +127,9 @@ suite('Editor ViewLayout - ViewLineParts', () => {
 		testCreateLineParts(
 			'        Hello world!        ',
 			[
-				new ViewLineToken(0, ''),
-				new ViewLineToken(8, 'a'),
-				new ViewLineToken(10, 'b')
+				new ViewLineToken(8, ''),
+				new ViewLineToken(10, 'a'),
+				new ViewLineToken(28, 'b')
 			],
 			0,
 			'boundary',
@@ -148,9 +149,9 @@ suite('Editor ViewLayout - ViewLineParts', () => {
 		testCreateLineParts(
 			'\t\tHello world!\t',
 			[
-				new ViewLineToken(0, ''),
-				new ViewLineToken(2, 'a'),
-				new ViewLineToken(4, 'b')
+				new ViewLineToken(2, ''),
+				new ViewLineToken(4, 'a'),
+				new ViewLineToken(15, 'b')
 			],
 			0,
 			'boundary',
@@ -169,9 +170,9 @@ suite('Editor ViewLayout - ViewLineParts', () => {
 		testCreateLineParts(
 			'  \t\t  Hello world! \t  \t   \t    ',
 			[
-				new ViewLineToken(0, ''),
-				new ViewLineToken(6, 'a'),
-				new ViewLineToken(8, 'b')
+				new ViewLineToken(6, ''),
+				new ViewLineToken(8, 'a'),
+				new ViewLineToken(31, 'b')
 			],
 			0,
 			'boundary',
@@ -195,9 +196,9 @@ suite('Editor ViewLayout - ViewLineParts', () => {
 		testCreateLineParts(
 			'\t\t  Hello world! \t  \t   \t    ',
 			[
-				new ViewLineToken(0, ''),
-				new ViewLineToken(4, 'a'),
-				new ViewLineToken(6, 'b')
+				new ViewLineToken(4, ''),
+				new ViewLineToken(6, 'a'),
+				new ViewLineToken(29, 'b')
 			],
 			2,
 			'boundary',
@@ -220,9 +221,9 @@ suite('Editor ViewLayout - ViewLineParts', () => {
 		testCreateLineParts(
 			'it  it it  it',
 			[
-				new ViewLineToken(0, ''),
-				new ViewLineToken(6, 'a'),
-				new ViewLineToken(7, 'b')
+				new ViewLineToken(6, ''),
+				new ViewLineToken(7, 'a'),
+				new ViewLineToken(13, 'b')
 			],
 			0,
 			'boundary',
@@ -244,9 +245,9 @@ suite('Editor ViewLayout - ViewLineParts', () => {
 		testCreateLineParts(
 			' Hello world!\t',
 			[
-				new ViewLineToken(0, ''),
-				new ViewLineToken(4, 'a'),
-				new ViewLineToken(6, 'b')
+				new ViewLineToken(4, ''),
+				new ViewLineToken(6, 'a'),
+				new ViewLineToken(14, 'b')
 			],
 			0,
 			'all',
@@ -266,7 +267,8 @@ suite('Editor ViewLayout - ViewLineParts', () => {
 	test('createLineParts can handle unsorted inline decorations', () => {
 		let actual = renderViewLine(new RenderLineInput(
 			'Hello world',
-			new ViewLineTokens([new ViewLineToken(0, '')], 0, 'Hello world'.length),
+			0,
+			[new ViewLineToken(11, '')],
 			[
 				new Decoration(5, 7, 'a'),
 				new Decoration(1, 3, 'b'),
@@ -369,7 +371,8 @@ suite('Editor ViewLayout - ViewLineParts', () => {
 	function createTestGetColumnOfLinePartOffset(lineContent: string, tabSize: number, parts: ViewLineToken[]): (partIndex: number, partLength: number, offset: number, expected: number) => void {
 		let renderLineOutput = renderViewLine(new RenderLineInput(
 			lineContent,
-			new ViewLineTokens(parts, 0, lineContent.length),
+			0,
+			parts,
 			[],
 			tabSize,
 			10,
@@ -390,7 +393,7 @@ suite('Editor ViewLayout - ViewLineParts', () => {
 			'hello world',
 			4,
 			[
-				new ViewLineToken(0, 'aToken')
+				new ViewLineToken(11, 'aToken')
 			]
 		);
 		testGetColumnOfLinePartOffset(0, 11, 0, 1);
@@ -412,12 +415,12 @@ suite('Editor ViewLayout - ViewLineParts', () => {
 			'var x = 3;',
 			4,
 			[
-				new ViewLineToken(0, 'meta type js storage var expr'),
-				new ViewLineToken(3, 'meta js var expr'),
-				new ViewLineToken(4, 'meta js var expr var-single-variable variable'),
-				new ViewLineToken(5, 'meta js var expr var-single-variable'),
-				new ViewLineToken(8, 'meta js var expr var-single-variable constant numeric'),
-				new ViewLineToken(9, ''),
+				new ViewLineToken(3, 'meta type js storage var expr'),
+				new ViewLineToken(4, 'meta js var expr'),
+				new ViewLineToken(5, 'meta js var expr var-single-variable variable'),
+				new ViewLineToken(8, 'meta js var expr var-single-variable'),
+				new ViewLineToken(9, 'meta js var expr var-single-variable constant numeric'),
+				new ViewLineToken(10, ''),
 			]
 		);
 		testGetColumnOfLinePartOffset(0, 3, 0, 1);
@@ -443,7 +446,7 @@ suite('Editor ViewLayout - ViewLineParts', () => {
 			'\t',
 			6,
 			[
-				new ViewLineToken(0, 'vs-whitespace')
+				new ViewLineToken(1, 'vs-whitespace')
 			]
 		);
 		testGetColumnOfLinePartOffset(0, 6, 0, 1);
@@ -460,8 +463,8 @@ suite('Editor ViewLayout - ViewLineParts', () => {
 			'\tfunction',
 			4,
 			[
-				new ViewLineToken(0, ''),
-				new ViewLineToken(1, 'meta type js function storage'),
+				new ViewLineToken(1, ''),
+				new ViewLineToken(9, 'meta type js function storage'),
 			]
 		);
 		testGetColumnOfLinePartOffset(0, 4, 0, 1);
@@ -485,8 +488,8 @@ suite('Editor ViewLayout - ViewLineParts', () => {
 			'\t\tfunction',
 			4,
 			[
-				new ViewLineToken(0, ''),
-				new ViewLineToken(2, 'meta type js function storage'),
+				new ViewLineToken(2, ''),
+				new ViewLineToken(10, 'meta type js function storage'),
 			]
 		);
 		testGetColumnOfLinePartOffset(0, 8, 0, 1);

src/vs/editor/test/common/viewLayout/viewLineRenderer.test.ts

@@ -6,19 +6,20 @@
 
 import * as assert from 'assert';
 import { renderViewLine, RenderLineInput, CharacterMapping } from 'vs/editor/common/viewLayout/viewLineRenderer';
-import { ViewLineToken, ViewLineTokens } from 'vs/editor/common/core/viewLineToken';
+import { ViewLineToken } from 'vs/editor/common/core/viewLineToken';
 import { CharCode } from 'vs/base/common/charCode';
 
 suite('viewLineRenderer.renderLine', () => {
 
-	function createPart(startIndex: number, type: string): ViewLineToken {
-		return new ViewLineToken(startIndex, type);
+	function createPart(endIndex: number, type: string): ViewLineToken {
+		return new ViewLineToken(endIndex, type);
 	}
 
 	function assertCharacterReplacement(lineContent: string, tabSize: number, expected: string, expectedCharOffsetInPart: number[][]): void {
 		let _actual = renderViewLine(new RenderLineInput(
 			lineContent,
-			new ViewLineTokens([new ViewLineToken(0, '')], 0, lineContent.length),
+			0,
+			[new ViewLineToken(lineContent.length, '')],
 			[],
 			tabSize,
 			0,
@@ -61,7 +62,8 @@ suite('viewLineRenderer.renderLine', () => {
 	function assertParts(lineContent: string, tabSize: number, parts: ViewLineToken[], expected: string, expectedCharOffsetInPart: number[][]): void {
 		let _actual = renderViewLine(new RenderLineInput(
 			lineContent,
-			new ViewLineTokens(parts, 0, lineContent.length),
+			0,
+			parts,
 			[],
 			tabSize,
 			0,
@@ -79,34 +81,35 @@ suite('viewLineRenderer.renderLine', () => {
 	});
 
 	test('uses part type', () => {
-		assertParts('x', 4, [createPart(0, 'y')], '<span class="y">x</span>', [[0, 1]]);
-		assertParts('x', 4, [createPart(0, 'aAbBzZ0123456789-cC')], '<span class="aAbBzZ0123456789-cC">x</span>', [[0, 1]]);
-		assertParts('x', 4, [createPart(0, '             ')], '<span class="             ">x</span>', [[0, 1]]);
+		assertParts('x', 4, [createPart(1, 'y')], '<span class="y">x</span>', [[0, 1]]);
+		assertParts('x', 4, [createPart(1, 'aAbBzZ0123456789-cC')], '<span class="aAbBzZ0123456789-cC">x</span>', [[0, 1]]);
+		assertParts('x', 4, [createPart(1, '             ')], '<span class="             ">x</span>', [[0, 1]]);
 	});
 
 	test('two parts', () => {
-		assertParts('xy', 4, [createPart(0, 'a'), createPart(1, 'b')], '<span class="a">x</span><span class="b">y</span>', [[0], [0, 1]]);
-		assertParts('xyz', 4, [createPart(0, 'a'), createPart(1, 'b')], '<span class="a">x</span><span class="b">yz</span>', [[0], [0, 1, 2]]);
-		assertParts('xyz', 4, [createPart(0, 'a'), createPart(2, 'b')], '<span class="a">xy</span><span class="b">z</span>', [[0, 1], [0, 1]]);
+		assertParts('xy', 4, [createPart(1, 'a'), createPart(2, 'b')], '<span class="a">x</span><span class="b">y</span>', [[0], [0, 1]]);
+		assertParts('xyz', 4, [createPart(1, 'a'), createPart(3, 'b')], '<span class="a">x</span><span class="b">yz</span>', [[0], [0, 1, 2]]);
+		assertParts('xyz', 4, [createPart(2, 'a'), createPart(3, 'b')], '<span class="a">xy</span><span class="b">z</span>', [[0, 1], [0, 1]]);
 	});
 
 	test('overflow', () => {
 		let _actual = renderViewLine(new RenderLineInput(
 			'Hello world!',
-			new ViewLineTokens([
-				createPart(0, '0'),
-				createPart(1, '1'),
-				createPart(2, '2'),
-				createPart(3, '3'),
-				createPart(4, '4'),
-				createPart(5, '5'),
-				createPart(6, '6'),
-				createPart(7, '7'),
-				createPart(8, '8'),
-				createPart(9, '9'),
-				createPart(10, '10'),
-				createPart(11, '11'),
-			], 0, 'Hello world!'.length),
+			0,
+			[
+				createPart(1, '0'),
+				createPart(2, '1'),
+				createPart(3, '2'),
+				createPart(4, '3'),
+				createPart(5, '4'),
+				createPart(6, '5'),
+				createPart(7, '6'),
+				createPart(8, '7'),
+				createPart(9, '8'),
+				createPart(10, '9'),
+				createPart(11, '10'),
+				createPart(12, '11'),
+			],
 			[],
 			4,
 			10,
@@ -139,18 +142,18 @@ suite('viewLineRenderer.renderLine', () => {
 	test('typical line', () => {
 		let lineText = '\t    export class Game { // http://test.com     ';
 		let lineParts = [
-			createPart(0, 'block meta ts'),
-			createPart(5, 'block declaration meta modifier object storage ts'),
-			createPart(11, 'block declaration meta object ts'),
-			createPart(12, 'block declaration meta object storage type ts'),
-			createPart(17, 'block declaration meta object ts'),
-			createPart(18, 'block class declaration entity meta name object ts'),
-			createPart(22, 'block declaration meta object ts'),
-			createPart(23, 'delimiter curly typescript'),
-			createPart(24, 'block body declaration meta object ts'),
-			createPart(25, 'block body comment declaration line meta object ts'),
-			createPart(28, 'block body comment declaration line meta object ts detected-link'),
-			createPart(43, 'block body comment declaration line meta object ts'),
+			createPart(5, 'block meta ts'),
+			createPart(11, 'block declaration meta modifier object storage ts'),
+			createPart(12, 'block declaration meta object ts'),
+			createPart(17, 'block declaration meta object storage type ts'),
+			createPart(18, 'block declaration meta object ts'),
+			createPart(22, 'block class declaration entity meta name object ts'),
+			createPart(23, 'block declaration meta object ts'),
+			createPart(24, 'delimiter curly typescript'),
+			createPart(25, 'block body declaration meta object ts'),
+			createPart(28, 'block body comment declaration line meta object ts'),
+			createPart(43, 'block body comment declaration line meta object ts detected-link'),
+			createPart(48, 'block body comment declaration line meta object ts'),
 		];
 		let expectedOutput = [
 			'<span class="vs-whitespace" style="width:40px">&rarr;&nbsp;&nbsp;&nbsp;</span>',
@@ -187,7 +190,8 @@ suite('viewLineRenderer.renderLine', () => {
 
 		let _actual = renderViewLine(new RenderLineInput(
 			lineText,
-			new ViewLineTokens(lineParts, 0, lineText.length),
+			0,
+			lineParts,
 			[],
 			4,
 			10,
@@ -204,16 +208,16 @@ suite('viewLineRenderer.renderLine', () => {
 		let lineText = '\t\t\tcursorStyle:\t\t\t\t\t\t(prevOpts.cursorStyle !== newOpts.cursorStyle),';
 
 		let lineParts = [
-			createPart(0, 'block body decl declaration meta method object ts'), // 3 chars
-			createPart(3, 'block body decl declaration member meta method object ts'), // 12 chars
-			createPart(15, 'block body decl declaration member meta method object ts'), // 6 chars
-			createPart(21, 'delimiter paren typescript'), // 1 char
-			createPart(22, 'block body decl declaration member meta method object ts'), // 21 chars
-			createPart(43, 'block body comparison decl declaration keyword member meta method object operator ts'), // 2 chars
-			createPart(45, 'block body comparison decl declaration keyword member meta method object operator ts'), // 1 char
-			createPart(46, 'block body decl declaration member meta method object ts'), // 20 chars
-			createPart(66, 'delimiter paren typescript'), // 1 char
-			createPart(67, 'block body decl declaration meta method object ts'), // 2 chars
+			createPart(3, 'block body decl declaration meta method object ts'), // 3 chars
+			createPart(15, 'block body decl declaration member meta method object ts'), // 12 chars
+			createPart(21, 'block body decl declaration member meta method object ts'), // 6 chars
+			createPart(22, 'delimiter paren typescript'), // 1 char
+			createPart(43, 'block body decl declaration member meta method object ts'), // 21 chars
+			createPart(45, 'block body comparison decl declaration keyword member meta method object operator ts'), // 2 chars
+			createPart(46, 'block body comparison decl declaration keyword member meta method object operator ts'), // 1 char
+			createPart(66, 'block body decl declaration member meta method object ts'), // 20 chars
+			createPart(67, 'delimiter paren typescript'), // 1 char
+			createPart(68, 'block body decl declaration meta method object ts'), // 2 chars
 		];
 		let expectedOutput = [
 			'<span class="block body decl declaration meta method object ts">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span>',
@@ -242,7 +246,8 @@ suite('viewLineRenderer.renderLine', () => {
 
 		let _actual = renderViewLine(new RenderLineInput(
 			lineText,
-			new ViewLineTokens(lineParts, 0, lineText.length),
+			0,
+			lineParts,
 			[],
 			4,
 			10,
@@ -259,16 +264,16 @@ suite('viewLineRenderer.renderLine', () => {
 		let lineText = ' \t\t\tcursorStyle:\t\t\t\t\t\t(prevOpts.cursorStyle !== newOpts.cursorStyle),';
 
 		let lineParts = [
-			createPart(0, 'block body decl declaration meta method object ts'), // 4 chars
-			createPart(4, 'block body decl declaration member meta method object ts'), // 12 chars
-			createPart(16, 'block body decl declaration member meta method object ts'), // 6 chars
-			createPart(22, 'delimiter paren typescript'), // 1 char
-			createPart(23, 'block body decl declaration member meta method object ts'), // 21 chars
-			createPart(44, 'block body comparison decl declaration keyword member meta method object operator ts'), // 2 chars
-			createPart(46, 'block body comparison decl declaration keyword member meta method object operator ts'), // 1 char
-			createPart(47, 'block body decl declaration member meta method object ts'), // 20 chars
-			createPart(67, 'delimiter paren typescript'), // 1 char
-			createPart(68, 'block body decl declaration meta method object ts'), // 2 chars
+			createPart(4, 'block body decl declaration meta method object ts'), // 4 chars
+			createPart(16, 'block body decl declaration member meta method object ts'), // 12 chars
+			createPart(22, 'block body decl declaration member meta method object ts'), // 6 chars
+			createPart(23, 'delimiter paren typescript'), // 1 char
+			createPart(44, 'block body decl declaration member meta method object ts'), // 21 chars
+			createPart(46, 'block body comparison decl declaration keyword member meta method object operator ts'), // 2 chars
+			createPart(47, 'block body comparison decl declaration keyword member meta method object operator ts'), // 1 char
+			createPart(67, 'block body decl declaration member meta method object ts'), // 20 chars
+			createPart(68, 'delimiter paren typescript'), // 1 char
+			createPart(69, 'block body decl declaration meta method object ts'), // 2 chars
 		];
 		let expectedOutput = [
 			'<span class="block body decl declaration meta method object ts">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span>',
@@ -297,7 +302,8 @@ suite('viewLineRenderer.renderLine', () => {
 
 		let _actual = renderViewLine(new RenderLineInput(
 			lineText,
-			new ViewLineTokens(lineParts, 0, lineText.length),
+			0,
+			lineParts,
 			[],
 			4,
 			10,