Reduce usage of ViewLineToken

src/vs/editor/browser/viewParts/minimap/minimap.ts

@@ -868,10 +868,9 @@ export class Minimap extends ViewPart {
 		let charIndex = 0;
 		let tabsCharDelta = 0;
 
-		for (let tokenIndex = 0, tokensLen = tokens.length; tokenIndex < tokensLen; tokenIndex++) {
-			const token = tokens[tokenIndex];
-			const tokenEndIndex = token.endIndex;
-			const tokenColorId = token.getForeground();
+		for (let tokenIndex = 0, tokensLen = tokens.getCount(); tokenIndex < tokensLen; tokenIndex++) {
+			const tokenEndIndex = tokens.getEndIndex(tokenIndex);
+			const tokenColorId = tokens.getForeground(tokenIndex);
 			const tokenColor = colorTracker.getColor(tokenColorId);
 
 			for (; charIndex < tokenEndIndex; charIndex++) {
@@ -927,4 +926,4 @@ registerThemingParticipant((theme, collector) => {
 	if (shadow) {
 		collector.addRule(`.monaco-editor .minimap-shadow-visible { box-shadow: ${shadow} -6px 0 6px -6px inset; }`);
 	}
-});
\ No newline at end of file
+});

src/vs/editor/browser/widget/diffEditorWidget.ts

@@ -24,7 +24,7 @@ import { LineDecoration } from 'vs/editor/common/viewLayout/lineDecorations';
 import { renderViewLine, RenderLineInput } from 'vs/editor/common/viewLayout/viewLineRenderer';
 import * as editorBrowser from 'vs/editor/browser/editorBrowser';
 import { CodeEditor } from 'vs/editor/browser/codeEditor';
-import { ViewLineToken } from 'vs/editor/common/core/viewLineToken';
+import { ViewLineToken, ViewLineTokens } from 'vs/editor/common/core/viewLineToken';
 import { Configuration } from 'vs/editor/browser/config/configuration';
 import { Position, IPosition } from 'vs/editor/common/core/position';
 import { Selection, ISelection } from 'vs/editor/common/core/selection';
@@ -1971,7 +1971,7 @@ class InlineViewZonesComputer extends ViewZonesComputer {
 			lineContent,
 			originalModel.mightContainRTL(),
 			0,
-			[new ViewLineToken(lineContent.length, defaultMetadata)],
+			new ViewLineTokens([new ViewLineToken(lineContent.length, defaultMetadata)]),
 			actualDecorations,
 			tabSize,
 			config.fontInfo.spaceWidth,

src/vs/editor/browser/widget/diffReview.ts

@@ -11,7 +11,7 @@ import * as dom from 'vs/base/browser/dom';
 import { FastDomNode, createFastDomNode } from 'vs/base/browser/fastDomNode';
 import * as editorCommon from 'vs/editor/common/editorCommon';
 import { renderViewLine2 as renderViewLine, RenderLineInput } from 'vs/editor/common/viewLayout/viewLineRenderer';
-import { ViewLineToken } from 'vs/editor/common/core/viewLineToken';
+import { ViewLineToken, ViewLineTokens } from 'vs/editor/common/core/viewLineToken';
 import { Configuration } from 'vs/editor/browser/config/configuration';
 import { Position } from 'vs/editor/common/core/position';
 import { ColorId, MetadataConsts, FontStyle } from 'vs/editor/common/modes';
@@ -736,7 +736,7 @@ export class DiffReview extends Disposable {
 			lineContent,
 			model.mightContainRTL(),
 			0,
-			[new ViewLineToken(lineContent.length, defaultMetadata)],
+			new ViewLineTokens([new ViewLineToken(lineContent.length, defaultMetadata)]),
 			[],
 			tabSize,
 			config.fontInfo.spaceWidth,

src/vs/editor/common/core/lineTokens.ts

@@ -5,7 +5,7 @@
 'use strict';
 
 import { TokenMetadata } from 'vs/editor/common/model/tokensBinaryEncoding';
-import { ViewLineTokenFactory, ViewLineToken } from 'vs/editor/common/core/viewLineToken';
+import { ViewLineTokenFactory, ViewLineToken, ViewLineTokens } from 'vs/editor/common/core/viewLineToken';
 import { ColorId, FontStyle, StandardTokenType, LanguageId } from 'vs/editor/common/modes';
 
 export class LineToken {
@@ -169,14 +169,22 @@ export class LineTokens {
 		return this.tokenAt(this._tokensCount - 1);
 	}
 
-	public inflate(): ViewLineToken[] {
+	private _inflate(): ViewLineToken[] {
 		return ViewLineTokenFactory.inflateArr(this._tokens);
 	}
 
-	public sliceAndInflate(startOffset: number, endOffset: number, deltaOffset: number): ViewLineToken[] {
+	public inflate(): ViewLineTokens {
+		return new ViewLineTokens(this._inflate());
+	}
+
+	private _sliceAndInflate(startOffset: number, endOffset: number, deltaOffset: number): ViewLineToken[] {
 		return ViewLineTokenFactory.sliceAndInflate(this._tokens, startOffset, endOffset, deltaOffset, this._textLength);
 	}
 
+	public sliceAndInflate(startOffset: number, endOffset: number, deltaOffset: number): ViewLineTokens {
+		return new ViewLineTokens(this._sliceAndInflate(startOffset, endOffset, deltaOffset));
+	}
+
 	public static convertToEndOffset(tokens: Uint32Array, lineTextLength: number): void {
 		const tokenCount = (tokens.length >>> 1);
 		const lastTokenIndex = tokenCount - 1;

src/vs/editor/common/core/viewLineToken.ts

@@ -58,6 +58,39 @@ export class ViewLineToken {
 	}
 }
 
+export class ViewLineTokens {
+
+	private readonly _actual: ViewLineToken[];
+
+	constructor(actual: ViewLineToken[]) {
+		this._actual = actual;
+	}
+
+	public equals(other: ViewLineTokens): boolean {
+		return ViewLineToken.equalsArr(this._actual, other._actual);
+	}
+
+	public getCount(): number {
+		return this._actual.length;
+	}
+
+	public getForeground(tokenIndex: number): ColorId {
+		return this._actual[tokenIndex].getForeground();
+	}
+
+	public getEndIndex(tokenIndex: number): number {
+		return this._actual[tokenIndex].endIndex;
+	}
+
+	public getType(tokenIndex: number): string {
+		return this._actual[tokenIndex].getType();
+	}
+
+	public getInlineStyle(tokenIndex: number, colorMap: string[]): string {
+		return this._actual[tokenIndex].getInlineStyle(colorMap);
+	}
+}
+
 export class ViewLineTokenFactory {
 
 	public static inflateArr(tokens: Uint32Array): ViewLineToken[] {

src/vs/editor/common/modes/textToHtmlTokenizer.ts

@@ -9,22 +9,21 @@ import { IState, ITokenizationSupport, TokenizationRegistry, LanguageId } from '
 import { NULL_STATE, nullTokenize2 } from 'vs/editor/common/modes/nullMode';
 import { LineTokens } from 'vs/editor/common/core/lineTokens';
 import { CharCode } from 'vs/base/common/charCode';
-import { ViewLineToken } from 'vs/editor/common/core/viewLineToken';
+import { ViewLineTokens } from 'vs/editor/common/core/viewLineToken';
 
 export function tokenizeToString(text: string, languageId: string): string {
 	return _tokenizeToString(text, _getSafeTokenizationSupport(languageId));
 }
 
-export function tokenizeLineToHTML(text: string, viewLineTokens: ViewLineToken[], colorMap: string[], startOffset: number, endOffset: number, tabSize: number): string {
+export function tokenizeLineToHTML(text: string, viewLineTokens: ViewLineTokens, colorMap: string[], startOffset: number, endOffset: number, tabSize: number): string {
 	let result = `<div>`;
 	let charIndex = startOffset;
 	let tabsCharDelta = 0;
 
-	for (let tokenIndex = 0, lenJ = viewLineTokens.length; tokenIndex < lenJ; tokenIndex++) {
-		const token = viewLineTokens[tokenIndex];
-		const tokenEndIndex = token.endIndex;
+	for (let tokenIndex = 0, tokenCount = viewLineTokens.getCount(); tokenIndex < tokenCount; tokenIndex++) {
+		const tokenEndIndex = viewLineTokens.getEndIndex(tokenIndex);
 
-		if (token.endIndex <= startOffset) {
+		if (tokenEndIndex <= startOffset) {
 			continue;
 		}
 
@@ -74,9 +73,9 @@ export function tokenizeLineToHTML(text: string, viewLineTokens: ViewLineToken[]
 			}
 		}
 
-		result += `<span style="${token.getInlineStyle(colorMap)}">${partContent}</span>`;
+		result += `<span style="${viewLineTokens.getInlineStyle(tokenIndex, colorMap)}">${partContent}</span>`;
 
-		if (token.endIndex > endOffset || charIndex >= endOffset) {
+		if (tokenEndIndex > endOffset || charIndex >= endOffset) {
 			break;
 		}
 	}
@@ -114,10 +113,11 @@ function _tokenizeToString(text: string, tokenizationSupport: ITokenizationSuppo
 		let viewLineTokens = lineTokens.inflate();
 
 		let startOffset = 0;
-		for (let j = 0, lenJ = viewLineTokens.length; j < lenJ; j++) {
-			const viewLineToken = viewLineTokens[j];
-			result += `<span class="${viewLineToken.getType()}">${strings.escape(line.substring(startOffset, viewLineToken.endIndex))}</span>`;
-			startOffset = viewLineToken.endIndex;
+		for (let j = 0, lenJ = viewLineTokens.getCount(); j < lenJ; j++) {
+			const type = viewLineTokens.getType(j);
+			const endIndex = viewLineTokens.getEndIndex(j);
+			result += `<span class="${type}">${strings.escape(line.substring(startOffset, endIndex))}</span>`;
+			startOffset = endIndex;
 		}
 
 		currentState = tokenizationResult.endState;

src/vs/editor/common/viewLayout/viewLineRenderer.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
-import { ViewLineToken } from 'vs/editor/common/core/viewLineToken';
+import { ViewLineTokens } from 'vs/editor/common/core/viewLineToken';
 import { CharCode } from 'vs/base/common/charCode';
 import { LineDecoration, LineDecorationsNormalizer } from 'vs/editor/common/viewLayout/lineDecorations';
 import * as strings from 'vs/base/common/strings';
@@ -32,31 +32,6 @@ class LinePart {
 	}
 }
 
-export class ViewLineTokens {
-
-	private readonly _actual: ViewLineToken[];
-
-	constructor(actual: ViewLineToken[]) {
-		this._actual = actual;
-	}
-
-	public equals(other: ViewLineTokens): boolean {
-		return ViewLineToken.equalsArr(this._actual, other._actual);
-	}
-
-	public getCount(): number {
-		return this._actual.length;
-	}
-
-	public getEndIndex(tokenIndex: number): number {
-		return this._actual[tokenIndex].endIndex;
-	}
-
-	public getType(tokenIndex: number): string {
-		return this._actual[tokenIndex].getType();
-	}
-}
-
 export class RenderLineInput {
 
 	public readonly useMonospaceOptimizations: boolean;
@@ -77,7 +52,7 @@ export class RenderLineInput {
 		lineContent: string,
 		mightContainRTL: boolean,
 		fauxIndentLength: number,
-		lineTokens: ViewLineToken[],
+		lineTokens: ViewLineTokens,
 		lineDecorations: LineDecoration[],
 		tabSize: number,
 		spaceWidth: number,
@@ -90,7 +65,7 @@ export class RenderLineInput {
 		this.lineContent = lineContent;
 		this.mightContainRTL = mightContainRTL;
 		this.fauxIndentLength = fauxIndentLength;
-		this.lineTokens = new ViewLineTokens(lineTokens);
+		this.lineTokens = lineTokens;
 		this.lineDecorations = lineDecorations;
 		this.tabSize = tabSize;
 		this.spaceWidth = spaceWidth;

src/vs/editor/common/viewModel/viewModel.ts

@@ -5,7 +5,7 @@
 'use strict';
 
 import { INewScrollPosition, EndOfLinePreference, IViewState, IModelDecorationOptions } from 'vs/editor/common/editorCommon';
-import { ViewLineToken } from 'vs/editor/common/core/viewLineToken';
+import { ViewLineTokens } from 'vs/editor/common/core/viewLineToken';
 import { Position, IPosition } from 'vs/editor/common/core/position';
 import { Range } from 'vs/editor/common/core/range';
 import { ViewEvent, IViewEventListener } from 'vs/editor/common/view/viewEvents';
@@ -180,13 +180,13 @@ export class ViewLineData {
 	/**
 	 * The tokens at this view line.
 	 */
-	public readonly tokens: ViewLineToken[];
+	public readonly tokens: ViewLineTokens;
 
 	constructor(
 		content: string,
 		minColumn: number,
 		maxColumn: number,
-		tokens: ViewLineToken[]
+		tokens: ViewLineTokens
 	) {
 		this.content = content;
 		this.minColumn = minColumn;
@@ -219,7 +219,7 @@ export class ViewLineRenderingData {
 	/**
 	 * The tokens at this view line.
 	 */
-	public readonly tokens: ViewLineToken[];
+	public readonly tokens: ViewLineTokens;
 	/**
 	 * Inline decorations at this view line.
 	 */
@@ -235,7 +235,7 @@ export class ViewLineRenderingData {
 		content: string,
 		mightContainRTL: boolean,
 		mightContainNonBasicASCII: boolean,
-		tokens: ViewLineToken[],
+		tokens: ViewLineTokens,
 		inlineDecorations: InlineDecoration[],
 		tabSize: number
 	) {

src/vs/editor/standalone/browser/colorizer.ts

@@ -10,7 +10,7 @@ import { IModel } from 'vs/editor/common/editorCommon';
 import { ColorId, MetadataConsts, FontStyle, TokenizationRegistry, ITokenizationSupport } from 'vs/editor/common/modes';
 import { IModeService } from 'vs/editor/common/services/modeService';
 import { renderViewLine2 as renderViewLine, RenderLineInput } from 'vs/editor/common/viewLayout/viewLineRenderer';
-import { ViewLineToken } from 'vs/editor/common/core/viewLineToken';
+import { ViewLineToken, ViewLineTokens } from 'vs/editor/common/core/viewLineToken';
 import { LineTokens } from 'vs/editor/common/core/lineTokens';
 import * as strings from 'vs/base/common/strings';
 import { IStandaloneThemeService } from 'vs/editor/standalone/common/standaloneThemeService';
@@ -94,7 +94,7 @@ export class Colorizer {
 		});
 	}
 
-	public static colorizeLine(line: string, mightContainRTL: boolean, tokens: ViewLineToken[], tabSize: number = 4): string {
+	public static colorizeLine(line: string, mightContainRTL: boolean, tokens: ViewLineTokens, tabSize: number = 4): string {
 		let renderResult = renderViewLine(new RenderLineInput(
 			false,
 			line,
@@ -142,7 +142,7 @@ function _fakeColorize(lines: string[], tabSize: number): string {
 			line,
 			false,
 			0,
-			[new ViewLineToken(line.length, defaultMetadata)],
+			new ViewLineTokens([new ViewLineToken(line.length, defaultMetadata)]),
 			[],
 			tabSize,
 			0,

src/vs/editor/test/common/core/lineTokens.test.ts

@@ -8,7 +8,7 @@
 import * as assert from 'assert';
 import { LineTokens } from 'vs/editor/common/core/lineTokens';
 import { MetadataConsts } from 'vs/editor/common/modes';
-import { ViewLineToken } from 'vs/editor/common/core/viewLineToken';
+import { ViewLineTokens } from 'vs/editor/common/core/viewLineToken';
 
 suite('LineTokens', () => {
 
@@ -205,13 +205,15 @@ suite('LineTokens', () => {
 		foreground: number;
 	}
 
-	function assertViewLineTokens(actual: ViewLineToken[], expected: ITestViewLineToken[]): void {
-		assert.deepEqual(actual.map(token => {
-			return {
-				endIndex: token.endIndex,
-				foreground: token.getForeground()
+	function assertViewLineTokens(_actual: ViewLineTokens, expected: ITestViewLineToken[]): void {
+		let actual: ITestViewLineToken[] = [];
+		for (let i = 0, len = _actual.getCount(); i < len; i++) {
+			actual[i] = {
+				endIndex: _actual.getEndIndex(i),
+				foreground: _actual.getForeground(i)
 			};
-		}), expected);
+		}
+		assert.deepEqual(actual, expected);
 	}
 
 	test('inflate', () => {

src/vs/editor/test/common/model/model.line.test.ts

@@ -7,25 +7,36 @@
 import * as assert from 'assert';
 import { LineTokens } from 'vs/editor/common/core/lineTokens';
 import { ModelLine, ILineEdit, computeIndentLevel } from 'vs/editor/common/model/modelLine';
-import { MetadataConsts, LanguageIdentifier } from 'vs/editor/common/modes';
+import { LanguageIdentifier, MetadataConsts } from 'vs/editor/common/modes';
 import { Range } from 'vs/editor/common/core/range';
 import { ViewLineToken, ViewLineTokenFactory } from 'vs/editor/common/core/viewLineToken';
 import { EditableTextModel } from 'vs/editor/common/model/editableTextModel';
 import { TextModel } from 'vs/editor/common/model/textModel';
 import { RawTextSource } from 'vs/editor/common/model/textSource';
 
-function assertLineTokens(_actual: LineTokens, _expected: TestToken[]): void {
+function assertLineTokens(__actual: LineTokens, _expected: TestToken[]): void {
 	let tmp = TestToken.toTokens(_expected);
-	LineTokens.convertToEndOffset(tmp, _actual.getLineContent().length);
+	LineTokens.convertToEndOffset(tmp, __actual.getLineContent().length);
 	let expected = ViewLineTokenFactory.inflateArr(tmp);
-	let actual = _actual.inflate();
+	let _actual = __actual.inflate();
+	interface ITestToken {
+		endIndex: number;
+		type: string;
+	}
+	let actual: ITestToken[] = [];
+	for (let i = 0, len = _actual.getCount(); i < len; i++) {
+		actual[i] = {
+			endIndex: _actual.getEndIndex(i),
+			type: _actual.getType(i)
+		};
+	}
 	let decode = (token: ViewLineToken) => {
 		return {
 			endIndex: token.endIndex,
 			type: token.getType()
 		};
 	};
-	assert.deepEqual(actual.map(decode), expected.map(decode));
+	assert.deepEqual(actual, expected.map(decode));
 }
 
 suite('ModelLine - getIndentLevel', () => {

src/vs/editor/test/common/model/textModelWithTokens.test.ts

@@ -268,14 +268,25 @@ suite('TextModelWithTokens regression tests', () => {
 			if (forceTokenization) {
 				model.forceTokenization(lineNumber);
 			}
-			let actual = model.getLineTokens(lineNumber).inflate();
+			let _actual = model.getLineTokens(lineNumber).inflate();
+			interface ISimpleViewToken {
+				endIndex: number;
+				foreground: number;
+			}
+			let actual: ISimpleViewToken[] = [];
+			for (let i = 0, len = _actual.getCount(); i < len; i++) {
+				actual[i] = {
+					endIndex: _actual.getEndIndex(i),
+					foreground: _actual.getForeground(i)
+				};
+			}
 			let decode = (token: ViewLineToken) => {
 				return {
 					endIndex: token.endIndex,
 					foreground: token.getForeground()
 				};
 			};
-			assert.deepEqual(actual.map(decode), expected.map(decode));
+			assert.deepEqual(actual, expected.map(decode));
 		}
 
 		let _tokenId = 10;

src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts

@@ -9,7 +9,7 @@ import { TokenizationRegistry, IState, LanguageIdentifier, ColorId, FontStyle, M
 import { tokenizeToString, tokenizeLineToHTML } from 'vs/editor/common/modes/textToHtmlTokenizer';
 import { MockMode } from 'vs/editor/test/common/mocks/mockMode';
 import { TokenizationResult2 } from 'vs/editor/common/core/token';
-import { ViewLineToken } from 'vs/editor/common/core/viewLineToken';
+import { ViewLineToken, ViewLineTokens } from 'vs/editor/common/core/viewLineToken';
 
 suite('Editor Modes - textToHtmlTokenizer', () => {
 	function toStr(pieces: { className: string; text: string }[]): string {
@@ -67,7 +67,7 @@ suite('Editor Modes - textToHtmlTokenizer', () => {
 
 	test('tokenizeLineToHTML', () => {
 		const text = 'Ciao hello world!';
-		const lineTokens = [
+		const lineTokens = new ViewLineTokens([
 			new ViewLineToken(
 				4,
 				(
@@ -100,7 +100,7 @@ suite('Editor Modes - textToHtmlTokenizer', () => {
 					| ((FontStyle.Underline) << MetadataConsts.FONT_STYLE_OFFSET)
 				) >>> 0
 			)
-		];
+		]);
 		const colorMap = [null, '#000000', '#ffffff', '#ff0000', '#00ff00', '#0000ff'];
 
 		assert.equal(

src/vs/editor/test/common/viewLayout/viewLineRenderer.test.ts

@@ -6,19 +6,23 @@
 
 import * as assert from 'assert';
 import { renderViewLine2 as renderViewLine, RenderLineInput, CharacterMapping } from 'vs/editor/common/viewLayout/viewLineRenderer';
-import { ViewLineToken } from 'vs/editor/common/core/viewLineToken';
+import { ViewLineToken, ViewLineTokens } from 'vs/editor/common/core/viewLineToken';
 import { CharCode } from 'vs/base/common/charCode';
 import { MetadataConsts } from 'vs/editor/common/modes';
 import { LineDecoration } from 'vs/editor/common/viewLayout/lineDecorations';
 import { InlineDecorationType } from 'vs/editor/common/viewModel/viewModel';
 
-suite('viewLineRenderer.renderLine', () => {
+function createViewLineTokens(viewLineTokens: ViewLineToken[]): ViewLineTokens {
+	return new ViewLineTokens(viewLineTokens);
+}
 
-	function createPart(endIndex: number, foreground: number): ViewLineToken {
-		return new ViewLineToken(endIndex, (
-			foreground << MetadataConsts.FOREGROUND_OFFSET
-		) >>> 0);
-	}
+function createPart(endIndex: number, foreground: number): ViewLineToken {
+	return new ViewLineToken(endIndex, (
+		foreground << MetadataConsts.FOREGROUND_OFFSET
+	) >>> 0);
+}
+
+suite('viewLineRenderer.renderLine', () => {
 
 	function assertCharacterReplacement(lineContent: string, tabSize: number, expected: string, expectedCharOffsetInPart: number[][], expectedPartLengts: number[]): void {
 		let _actual = renderViewLine(new RenderLineInput(
@@ -26,7 +30,7 @@ suite('viewLineRenderer.renderLine', () => {
 			lineContent,
 			false,
 			0,
-			[new ViewLineToken(lineContent.length, 0)],
+			createViewLineTokens([new ViewLineToken(lineContent.length, 0)]),
 			[],
 			tabSize,
 			0,
@@ -72,7 +76,7 @@ suite('viewLineRenderer.renderLine', () => {
 			lineContent,
 			false,
 			0,
-			parts,
+			createViewLineTokens(parts),
 			[],
 			tabSize,
 			0,
@@ -108,7 +112,7 @@ suite('viewLineRenderer.renderLine', () => {
 			'Hello world!',
 			false,
 			0,
-			[
+			createViewLineTokens([
 				createPart(1, 0),
 				createPart(2, 1),
 				createPart(3, 2),
@@ -121,7 +125,7 @@ suite('viewLineRenderer.renderLine', () => {
 				createPart(10, 9),
 				createPart(11, 10),
 				createPart(12, 11),
-			],
+			]),
 			[],
 			4,
 			10,
@@ -157,7 +161,7 @@ suite('viewLineRenderer.renderLine', () => {
 
 	test('typical line', () => {
 		let lineText = '\t    export class Game { // http://test.com     ';
-		let lineParts = [
+		let lineParts = createViewLineTokens([
 			createPart(5, 1),
 			createPart(11, 2),
 			createPart(12, 3),
@@ -170,7 +174,7 @@ suite('viewLineRenderer.renderLine', () => {
 			createPart(28, 10),
 			createPart(43, 11),
 			createPart(48, 12),
-		];
+		]);
 		let expectedOutput = [
 			'<span class="vs-whitespace" style="width:40px">\u2192\u00a0\u00a0\u00a0</span>',
 			'<span class="vs-whitespace" style="width:40px">\u00b7\u00b7\u00b7\u00b7</span>',
@@ -226,7 +230,7 @@ suite('viewLineRenderer.renderLine', () => {
 	test('issue #2255: Weird line rendering part 1', () => {
 		let lineText = '\t\t\tcursorStyle:\t\t\t\t\t\t(prevOpts.cursorStyle !== newOpts.cursorStyle),';
 
-		let lineParts = [
+		let lineParts = createViewLineTokens([
 			createPart(3, 1), // 3 chars
 			createPart(15, 2), // 12 chars
 			createPart(21, 3), // 6 chars
@@ -237,7 +241,7 @@ suite('viewLineRenderer.renderLine', () => {
 			createPart(66, 8), // 20 chars
 			createPart(67, 9), // 1 char
 			createPart(68, 10), // 2 chars
-		];
+		]);
 		let expectedOutput = [
 			'<span class="mtk1">\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0</span>',
 			'<span class="mtk2">cursorStyle:</span>',
@@ -285,7 +289,7 @@ suite('viewLineRenderer.renderLine', () => {
 	test('issue #2255: Weird line rendering part 2', () => {
 		let lineText = ' \t\t\tcursorStyle:\t\t\t\t\t\t(prevOpts.cursorStyle !== newOpts.cursorStyle),';
 
-		let lineParts = [
+		let lineParts = createViewLineTokens([
 			createPart(4, 1), // 4 chars
 			createPart(16, 2), // 12 chars
 			createPart(22, 3), // 6 chars
@@ -296,7 +300,7 @@ suite('viewLineRenderer.renderLine', () => {
 			createPart(67, 8), // 20 chars
 			createPart(68, 9), // 1 char
 			createPart(69, 10), // 2 chars
-		];
+		]);
 		let expectedOutput = [
 			'<span class="mtk1">\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0</span>',
 			'<span class="mtk2">cursorStyle:</span>',
@@ -344,12 +348,12 @@ suite('viewLineRenderer.renderLine', () => {
 	test('issue Microsoft/monaco-editor#280: Improved source code rendering for RTL languages', () => {
 		let lineText = 'var קודמות = \"מיותר קודמות צ\'ט של, אם לשון העברית שינויים ויש, אם\";';
 
-		let lineParts = [
+		let lineParts = createViewLineTokens([
 			createPart(3, 6),
 			createPart(13, 1),
 			createPart(66, 20),
 			createPart(67, 1),
-		];
+		]);
 
 		let expectedOutput = [
 			'<span class="mtk6" dir="ltr">var</span>',
@@ -384,7 +388,7 @@ suite('viewLineRenderer.renderLine', () => {
 		let _lineText = 'This is just a long line that contains very interesting text. This is just a long line that contains very interesting text.';
 
 		function assertSplitsTokens(message: string, lineText: string, expectedOutput: string[]): void {
-			let lineParts = [createPart(lineText.length, 1)];
+			let lineParts = createViewLineTokens([createPart(lineText.length, 1)]);
 			let actual = renderViewLine(new RenderLineInput(
 				false,
 				lineText,
@@ -481,7 +485,7 @@ suite('viewLineRenderer.renderLine', () => {
 		let _lineText = 'This is just a long line that contains very interesting text. This is just a long line that contains very interesting text.';
 
 		function assertSplitsTokens(message: string, lineText: string, expectedOutput: string[]): void {
-			let lineParts = [createPart(lineText.length, 1)];
+			let lineParts = createViewLineTokens([createPart(lineText.length, 1)]);
 			let actual = renderViewLine(new RenderLineInput(
 				false,
 				lineText,
@@ -514,7 +518,7 @@ suite('viewLineRenderer.renderLine', () => {
 	test('issue #20624: Unaligned surrogate pairs are corrupted at multiples of 50 columns', () => {
 		let lineText = 'a𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷𠮷';
 
-		let lineParts = [createPart(lineText.length, 1)];
+		let lineParts = createViewLineTokens([createPart(lineText.length, 1)]);
 		let actual = renderViewLine(new RenderLineInput(
 			false,
 			lineText,
@@ -541,7 +545,7 @@ suite('viewLineRenderer.renderLine', () => {
 
 	test('issue #6885: Does not split large tokens in RTL text', () => {
 		let lineText = 'את גרמנית בהתייחסות שמו, שנתי המשפט אל חפש, אם כתב אחרים ולחבר. של התוכן אודות בויקיפדיה כלל, של עזרה כימיה היא. על עמוד יוצרים מיתולוגיה סדר, אם שכל שתפו לעברית שינויים, אם שאלות אנגלית עזה. שמות בקלות מה סדר.';
-		let lineParts = [createPart(lineText.length, 1)];
+		let lineParts = createViewLineTokens([createPart(lineText.length, 1)]);
 		let expectedOutput = [
 			'<span class="mtk1" dir="ltr">את\u00a0גרמנית\u00a0בהתייחסות\u00a0שמו,\u00a0שנתי\u00a0המשפט\u00a0אל\u00a0חפש,\u00a0אם\u00a0כתב\u00a0אחרים\u00a0ולחבר.\u00a0של\u00a0התוכן\u00a0אודות\u00a0בויקיפדיה\u00a0כלל,\u00a0של\u00a0עזרה\u00a0כימיה\u00a0היא.\u00a0על\u00a0עמוד\u00a0יוצרים\u00a0מיתולוגיה\u00a0סדר,\u00a0אם\u00a0שכל\u00a0שתפו\u00a0לעברית\u00a0שינויים,\u00a0אם\u00a0שאלות\u00a0אנגלית\u00a0עזה.\u00a0שמות\u00a0בקלות\u00a0מה\u00a0סדר.</span>'
 		];
@@ -566,7 +570,7 @@ suite('viewLineRenderer.renderLine', () => {
 	test('issue #19673: Monokai Theme bad-highlighting in line wrap', () => {
 		let lineText = '    MongoCallback<string>): void {';
 
-		let lineParts = [
+		let lineParts = createViewLineTokens([
 			createPart(17, 1),
 			createPart(18, 2),
 			createPart(24, 3),
@@ -575,7 +579,7 @@ suite('viewLineRenderer.renderLine', () => {
 			createPart(28, 6),
 			createPart(32, 7),
 			createPart(34, 8),
-		];
+		]);
 		let expectedOutput = [
 			'<span class="">\u00a0\u00a0\u00a0\u00a0</span>',
 			'<span class="mtk1">MongoCallback</span>',
@@ -666,11 +670,6 @@ suite('viewLineRenderer.renderLine', () => {
 });
 
 suite('viewLineRenderer.renderLine 2', () => {
-	function createPart(endIndex: number, foreground: number): ViewLineToken {
-		return new ViewLineToken(endIndex, (
-			foreground << MetadataConsts.FOREGROUND_OFFSET
-		) >>> 0);
-	}
 
 	function testCreateLineParts(fontIsMonospace: boolean, lineContent: string, tokens: ViewLineToken[], fauxIndentLength: number, renderWhitespace: 'none' | 'boundary' | 'all', expected: string): void {
 		let actual = renderViewLine(new RenderLineInput(
@@ -678,7 +677,7 @@ suite('viewLineRenderer.renderLine 2', () => {
 			lineContent,
 			false,
 			fauxIndentLength,
-			tokens,
+			createViewLineTokens(tokens),
 			[],
 			4,
 			10,
@@ -700,7 +699,7 @@ suite('viewLineRenderer.renderLine 2', () => {
 			lineContent,
 			false,
 			0,
-			[createPart(21, 3)],
+			createViewLineTokens([createPart(21, 3)]),
 			[new LineDecoration(1, 22, 'link', InlineDecorationType.Regular)],
 			4,
 			10,
@@ -728,13 +727,13 @@ suite('viewLineRenderer.renderLine 2', () => {
 			lineContent,
 			false,
 			0,
-			[
+			createViewLineTokens([
 				createPart(49, 6),
 				createPart(51, 4),
 				createPart(72, 6),
 				createPart(74, 4),
 				createPart(84, 6),
-			],
+			]),
 			[
 				new LineDecoration(13, 51, 'detected-link', InlineDecorationType.Regular)
 			],
@@ -992,7 +991,7 @@ suite('viewLineRenderer.renderLine 2', () => {
 			'Hello world',
 			false,
 			0,
-			[createPart(11, 0)],
+			createViewLineTokens([createPart(11, 0)]),
 			[
 				new LineDecoration(5, 7, 'a', InlineDecorationType.Regular),
 				new LineDecoration(1, 3, 'b', InlineDecorationType.Regular),
@@ -1033,7 +1032,7 @@ suite('viewLineRenderer.renderLine 2', () => {
 			lineContent,
 			false,
 			0,
-			[createPart(4, 3)],
+			createViewLineTokens([createPart(4, 3)]),
 			[new LineDecoration(1, 2, 'before', InlineDecorationType.Before)],
 			4,
 			10,
@@ -1062,7 +1061,7 @@ suite('viewLineRenderer.renderLine 2', () => {
 			lineContent,
 			false,
 			0,
-			[createPart(4, 3)],
+			createViewLineTokens([createPart(4, 3)]),
 			[new LineDecoration(2, 3, 'before', InlineDecorationType.Before)],
 			4,
 			10,
@@ -1092,7 +1091,7 @@ suite('viewLineRenderer.renderLine 2', () => {
 			lineContent,
 			false,
 			0,
-			[createPart(0, 3)],
+			createViewLineTokens([createPart(0, 3)]),
 			[new LineDecoration(1, 2, 'before', InlineDecorationType.Before)],
 			4,
 			10,
@@ -1118,7 +1117,7 @@ suite('viewLineRenderer.renderLine 2', () => {
 			'  1. 🙏',
 			false,
 			0,
-			[createPart(7, 3)],
+			createViewLineTokens([createPart(7, 3)]),
 			[new LineDecoration(7, 8, 'inline-folded', InlineDecorationType.After)],
 			2,
 			10,
@@ -1145,7 +1144,7 @@ suite('viewLineRenderer.renderLine 2', () => {
 			'',
 			false,
 			0,
-			[createPart(0, 3)],
+			createViewLineTokens([createPart(0, 3)]),
 			[
 				new LineDecoration(1, 2, 'before', InlineDecorationType.Before),
 				new LineDecoration(0, 1, 'after', InlineDecorationType.After),
@@ -1174,7 +1173,7 @@ suite('viewLineRenderer.renderLine 2', () => {
 			'\t}',
 			false,
 			0,
-			[createPart(2, 3)],
+			createViewLineTokens([createPart(2, 3)]),
 			[
 				new LineDecoration(3, 3, 'ced-TextEditorDecorationType2-5e9b9b3f-3 ced-TextEditorDecorationType2-3', InlineDecorationType.Before),
 				new LineDecoration(3, 3, 'ced-TextEditorDecorationType2-5e9b9b3f-4 ced-TextEditorDecorationType2-4', InlineDecorationType.After),
@@ -1203,7 +1202,7 @@ suite('viewLineRenderer.renderLine 2', () => {
 			lineContent,
 			false,
 			0,
-			parts,
+			createViewLineTokens(parts),
 			[],
 			tabSize,
 			10,

src/vs/editor/test/common/viewModel/splitLinesCollection.test.ts

@@ -16,7 +16,7 @@ import * as modes from 'vs/editor/common/modes';
 import { NULL_STATE } from 'vs/editor/common/modes/nullMode';
 import { TokenizationResult2 } from 'vs/editor/common/core/token';
 import { IDisposable } from 'vs/base/common/lifecycle';
-import { ViewLineToken } from 'vs/editor/common/core/viewLineToken';
+import { ViewLineTokens } from 'vs/editor/common/core/viewLineToken';
 import { ViewLineData } from 'vs/editor/common/viewModel/viewModel';
 import { Range } from 'vs/editor/common/core/range';
 
@@ -363,14 +363,15 @@ suite('SplitLinesCollection', () => {
 		value: number;
 	}
 
-	function assertViewLineTokens(actual: ViewLineToken[], expected: ITestViewLineToken[]): void {
-		let _actual = actual.map((token) => {
-			return {
-				endIndex: token.endIndex,
-				value: token.getForeground()
+	function assertViewLineTokens(_actual: ViewLineTokens, expected: ITestViewLineToken[]): void {
+		let actual: ITestViewLineToken[] = [];
+		for (let i = 0, len = _actual.getCount(); i < len; i++) {
+			actual[i] = {
+				endIndex: _actual.getEndIndex(i),
+				value: _actual.getForeground(i)
 			};
-		});
-		assert.deepEqual(_actual, expected);
+		}
+		assert.deepEqual(actual, expected);
 	}
 
 	interface ITestMinimapLineRenderingData {
@@ -785,4 +786,4 @@ function createModel(text: string): IModel {
 			return text.length + 1;
 		}
 	};
-}
\ No newline at end of file
+}