Optimize renderLine

scripts/code-perf.sh

+#!/usr/bin/env bash
+
+if [[ "$OSTYPE" == "darwin"* ]]; then
+	realpath() { [[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}"; }
+	ROOT=$(dirname $(dirname $(realpath "$0")))
+else
+	ROOT=$(dirname $(dirname $(readlink -f $0)))
+fi
+
+function code() {
+	cd $ROOT
+
+	# Node modules
+	test -d node_modules || ./scripts/npm.sh install
+
+	# Get electron
+	./node_modules/.bin/gulp electron
+
+	# Build
+	test -d out || ./node_modules/.bin/gulp compile
+
+	# Configuration
+	export NODE_ENV=development
+	export VSCODE_DEV=1
+	export ELECTRON_ENABLE_LOGGING=1
+	export ELECTRON_ENABLE_STACK_DUMPING=1
+
+	# Launch Code
+	if [[ "$OSTYPE" == "darwin"* ]]; then
+		exec ./.build/electron/Electron.app/Contents/MacOS/Electron . "$@"
+	else
+		exec ./.build/electron/electron . "$@" --no-sandbox --js-flags="--trace-hydrogen --trace-phase=Z --trace-deopt --code-comments --hydrogen-track-positions --redirect-code-traces"
+	fi
+}
+
+code "$@"

src/vs/editor/browser/standalone/colorizer.ts

@@ -6,10 +6,11 @@
 
 import {RunOnceScheduler} from 'vs/base/common/async';
 import {TPromise} from 'vs/base/common/winjs.base';
-import {ILineToken, IModel, LineTokensBinaryEncoding} from 'vs/editor/common/editorCommon';
+import {LineToken, IModel} from 'vs/editor/common/editorCommon';
 import {ILineTokens, IMode} from 'vs/editor/common/modes';
 import {IModeService} from 'vs/editor/common/services/modeService';
-import {IRenderLineOutput, renderLine} from 'vs/editor/common/viewLayout/viewLineRenderer';
+import {IRenderLineOutput, renderLine, RenderLineInput} from 'vs/editor/common/viewLayout/viewLineRenderer';
+import * as TokensBinaryEncoding from 'vs/editor/common/model/tokensBinaryEncoding';
 
 export interface IColorizerOptions {
 	tabSize?: number;
@@ -91,21 +92,21 @@ export class Colorizer {
 		return result;
 	}
 
-	public static colorizeLine(line:string, tokens:ILineToken[], tabSize:number = 4): string {
-		var renderResult = renderLine({
-			lineContent: line,
-			parts: tokens,
-			stopRenderingLineAfter: -1,
-			renderWhitespace: false,
-			tabSize: tabSize
-		});
+	public static colorizeLine(line:string, tokens:LineToken[], tabSize:number = 4): string {
+		var renderResult = renderLine(new RenderLineInput(
+			line,
+			tabSize,
+			-1,
+			false,
+			tokens
+		));
 		return renderResult.output.join('');
 	}
 
 	public static colorizeModelLine(model:IModel, lineNumber:number, tabSize:number = 4): string {
 		var content = model.getLineContent(lineNumber);
 		var tokens = model.getLineTokens(lineNumber, false);
-		var inflatedTokens = LineTokensBinaryEncoding.inflateArr(tokens.getBinaryEncodedTokensMap(), tokens.getBinaryEncodedTokens());
+		var inflatedTokens = TokensBinaryEncoding.inflateArr(tokens.getBinaryEncodedTokensMap(), tokens.getBinaryEncodedTokens());
 		return this.colorizeLine(content, inflatedTokens, tabSize);
 	}
 }
@@ -135,13 +136,13 @@ function actualColorize(lines:string[], mode:IMode, tabSize:number): IActualColo
 			retokenize.push(tokenizeResult.retokenize);
 		}
 
-		renderResult = renderLine({
-			lineContent: line,
-			parts: tokenizeResult.tokens,
-			stopRenderingLineAfter: -1,
-			renderWhitespace: false,
-			tabSize: tabSize
-		});
+		renderResult = renderLine(new RenderLineInput(
+			line,
+			tabSize,
+			-1,
+			false,
+			tokenizeResult.tokens.map(t => new LineToken(t.startIndex, t.type))
+		));
 
 		html = html.concat(renderResult.output);
 		html.push('<br/>');

src/vs/editor/browser/standalone/standaloneEditor.ts

@@ -12,6 +12,7 @@ import {ClassNames, ContentWidgetPositionPreference, OverlayWidgetPositionPrefer
 import {Colorizer} from 'vs/editor/browser/standalone/colorizer';
 import * as standaloneCodeEditor from 'vs/editor/browser/standalone/standaloneCodeEditor';
 import {ILanguageDef} from 'vs/editor/standalone-languages/types';
+import * as TokensBinaryEncoding from 'vs/editor/common/model/tokensBinaryEncoding';
 
 var global:any = self;
 if (!global.Monaco) {
@@ -41,7 +42,7 @@ Monaco.Editor.wrappingIndentFromString = editorCommon.wrappingIndentFromString;
 Monaco.Editor.OverviewRulerLane = editorCommon.OverviewRulerLane;
 Monaco.Editor.EndOfLinePreference = editorCommon.EndOfLinePreference;
 Monaco.Editor.EndOfLineSequence = editorCommon.EndOfLineSequence;
-Monaco.Editor.LineTokensBinaryEncoding = editorCommon.LineTokensBinaryEncoding;
+Monaco.Editor.LineTokensBinaryEncoding = TokensBinaryEncoding;
 Monaco.Editor.TrackedRangeStickiness = editorCommon.TrackedRangeStickiness;
 Monaco.Editor.VerticalRevealType = editorCommon.VerticalRevealType;
 Monaco.Editor.MouseTargetType = editorCommon.MouseTargetType;

src/vs/editor/browser/viewParts/lines/viewLine.ts

@@ -8,7 +8,7 @@ import * as browser from 'vs/base/browser/browser';
 import {StyleMutator} from 'vs/base/browser/styleMutator';
 import {HorizontalRange, IConfigurationChangedEvent, IModelDecoration} from 'vs/editor/common/editorCommon';
 import {ILineParts, createLineParts} from 'vs/editor/common/viewLayout/viewLineParts';
-import {renderLine} from 'vs/editor/common/viewLayout/viewLineRenderer';
+import {renderLine, RenderLineInput} from 'vs/editor/common/viewLayout/viewLineRenderer';
 import {ClassNames, IViewContext} from 'vs/editor/browser/editorBrowser';
 import {IVisibleLineData} from 'vs/editor/browser/view/viewLayer';
 
@@ -134,13 +134,13 @@ export class ViewLine implements IVisibleLineData {
 
 		this._cachedWidth = -1;
 
-		let r = renderLine({
-			lineContent: this._context.model.getLineContent(lineNumber),
-			tabSize: this._context.model.getTabSize(),
-			stopRenderingLineAfter: this._context.configuration.editor.stopRenderingLineAfter,
-			renderWhitespace: this._context.configuration.editor.renderWhitespace,
-			parts: lineParts.getParts()
-		});
+		let r = renderLine(new RenderLineInput(
+			this._context.model.getLineContent(lineNumber),
+			this._context.model.getTabSize(),
+			this._context.configuration.editor.stopRenderingLineAfter,
+			this._context.configuration.editor.renderWhitespace,
+			lineParts.getParts()
+		));
 
 		this._charOffsetInPart = r.charOffsetInPart;
 		this._lastRenderedPartIndex = r.lastRenderedPartIndex;

src/vs/editor/browser/widget/codeEditorWidget.ts

@@ -26,6 +26,7 @@ import * as editorBrowser from 'vs/editor/browser/editorBrowser';
 import {EditorBrowserRegistry} from 'vs/editor/browser/editorBrowserExtensions';
 import {Colorizer} from 'vs/editor/browser/standalone/colorizer';
 import {View} from 'vs/editor/browser/view/viewImpl';
+import * as TokensBinaryEncoding from 'vs/editor/common/model/tokensBinaryEncoding';
 
 export class CodeEditorWidget extends CommonCodeEditor implements editorBrowser.ICodeEditor {
 
@@ -97,7 +98,7 @@ export class CodeEditorWidget extends CommonCodeEditor implements editorBrowser.
 		}
 		var content = model.getLineContent(lineNumber);
 		var tokens = model.getLineTokens(lineNumber, false);
-		var inflatedTokens = editorCommon.LineTokensBinaryEncoding.inflateArr(tokens.getBinaryEncodedTokensMap(), tokens.getBinaryEncodedTokens());
+		var inflatedTokens = TokensBinaryEncoding.inflateArr(tokens.getBinaryEncodedTokensMap(), tokens.getBinaryEncodedTokens());
 		var tabSize = model.getOptions().tabSize;
 		return Colorizer.colorizeLine(content, inflatedTokens, tabSize);
 	}

src/vs/editor/browser/widget/diffEditorWidget.ts

@@ -20,7 +20,7 @@ import {Range} from 'vs/editor/common/core/range';
 import * as editorCommon from 'vs/editor/common/editorCommon';
 import {IEditorWorkerService} from 'vs/editor/common/services/editorWorkerService';
 import {ILineParts, createLineParts} from 'vs/editor/common/viewLayout/viewLineParts';
-import {renderLine} from 'vs/editor/common/viewLayout/viewLineRenderer';
+import {renderLine, RenderLineInput} from 'vs/editor/common/viewLayout/viewLineRenderer';
 import * as editorBrowser from 'vs/editor/browser/editorBrowser';
 import {CodeEditorWidget} from 'vs/editor/browser/widget/codeEditorWidget';
 
@@ -1766,7 +1766,7 @@ class InlineViewZonesComputer extends ViewZonesComputer {
 
 		lineTokens = {
 			getTokens: () => {
-				return [{ startIndex: 0, type: '' }];
+				return [new editorCommon.LineToken(0, '')];
 			},
 			getFauxIndentLength: () => {
 				return 0;
@@ -1784,13 +1784,13 @@ class InlineViewZonesComputer extends ViewZonesComputer {
 
 		parts = createLineParts(lineNumber, 1, lineContent, lineTokens, decorations, config.renderWhitespace);
 
-		var r = renderLine({
-			lineContent: lineContent,
-			tabSize: tabSize,
-			stopRenderingLineAfter: config.stopRenderingLineAfter,
-			renderWhitespace: config.renderWhitespace,
-			parts: parts.getParts()
-		});
+		var r = renderLine(new RenderLineInput(
+			lineContent,
+			tabSize,
+			config.stopRenderingLineAfter,
+			config.renderWhitespace,
+			parts.getParts()
+		));
 
 		var myResult:string[] = [];
 

src/vs/editor/common/editorCommon.ts

@@ -13,7 +13,6 @@ import {IDisposable} from 'vs/base/common/lifecycle';
 import URI from 'vs/base/common/uri';
 import {TPromise} from 'vs/base/common/winjs.base';
 import {IInstantiationService, IConstructorSignature1, IConstructorSignature2} from 'vs/platform/instantiation/common/instantiation';
-import * as TokensBinaryEncoding from 'vs/editor/common/model/tokensBinaryEncoding';
 import {ILineContext, IMode, IModeTransition, IToken} from 'vs/editor/common/modes';
 
 export type KeyCode = KeyCode;
@@ -1183,9 +1182,16 @@ export interface ICursorStateComputer {
 /**
  * A token on a line.
  */
-export interface ILineToken {
-	startIndex: number;
-	type: string;
+export class LineToken {
+	public _lineTokenTrait: void;
+
+	public startIndex:number;
+	public type:string;
+
+	constructor(startIndex:number, type:string) {
+		this.startIndex = +startIndex;// @perf
+		this.type = type.replace(/[^a-z0-9\-]/gi, ' ');
+	}
 }
 
 export interface ITokensInflatorMap {
@@ -1207,7 +1213,6 @@ export interface ILineTokensBinaryEncoding {
 	findIndexOfOffset(binaryEncodedTokens:number[], offset:number): number;
 	sliceAndInflate(map:ITokensInflatorMap, binaryEncodedTokens:number[], startOffset:number, endOffset:number, deltaStartIndex:number): IToken[];
 }
-export var LineTokensBinaryEncoding:ILineTokensBinaryEncoding = TokensBinaryEncoding;
 
 /**
  * A list of tokens on a line.
@@ -2509,7 +2514,7 @@ export interface IConfiguration {
 // --- view
 
 export interface IViewLineTokens {
-	getTokens(): ILineToken[];
+	getTokens(): LineToken[];
 	getFauxIndentLength(): number;
 	getTextLength(): number;
 	equals(other:IViewLineTokens): boolean;

src/vs/editor/common/model/modelLine.ts

@@ -5,8 +5,9 @@
 'use strict';
 
 import * as strings from 'vs/base/common/strings';
-import {ILineTokens, IReadOnlyLineMarker, ITokensInflatorMap, LineTokensBinaryEncoding} from 'vs/editor/common/editorCommon';
+import {ILineTokens, IReadOnlyLineMarker, ITokensInflatorMap} from 'vs/editor/common/editorCommon';
 import {IMode, IModeTransition, IState, IToken} from 'vs/editor/common/modes';
+import * as TokensBinaryEncoding from 'vs/editor/common/model/tokensBinaryEncoding';
 
 export interface ILineEdit {
 	startColumn: number;
@@ -181,7 +182,7 @@ export class ModelLine {
 
 		var lineTokens = this._lineTokens;
 
-		let BIN = LineTokensBinaryEncoding;
+		let BIN = TokensBinaryEncoding;
 		let tokens = lineTokens.getBinaryEncodedTokens();
 		let tokensLength = tokens.length;
 		let tokensIndex = 0;
@@ -241,7 +242,7 @@ export class ModelLine {
 		this.text = text;
 
 		if (this._lineTokens) {
-			let BIN = LineTokensBinaryEncoding,
+			let BIN = TokensBinaryEncoding,
 				map = this._lineTokens.getBinaryEncodedTokensMap(),
 				tokens = this._lineTokens.getBinaryEncodedTokens(),
 				lineTextLength = this.text.length;
@@ -492,7 +493,7 @@ export class ModelLine {
 
 			// Adjust other tokens
 			if (thisTextLength > 0) {
-				let BIN = LineTokensBinaryEncoding;
+				let BIN = TokensBinaryEncoding;
 
 				for (let i = 0, len = otherTokens.length; i < len; i++) {
 					let token = otherTokens[i];
@@ -673,7 +674,7 @@ function toLineTokensFromInflated(map:ITokensInflatorMap, tokens:IToken[], textL
 		}
 	}
 
-	let deflated = LineTokensBinaryEncoding.deflateArr(map, tokens);
+	let deflated = TokensBinaryEncoding.deflateArr(map, tokens);
 	return new LineTokens(map, deflated);
 }
 
@@ -692,9 +693,9 @@ function toLineTokensFromDeflated(map:ITokensInflatorMap, tokens:number[], textL
 	return new LineTokens(map, tokens);
 }
 
-var getStartIndex = LineTokensBinaryEncoding.getStartIndex;
-var getType = LineTokensBinaryEncoding.getType;
-var findIndexOfOffset = LineTokensBinaryEncoding.findIndexOfOffset;
+var getStartIndex = TokensBinaryEncoding.getStartIndex;
+var getType = TokensBinaryEncoding.getType;
+var findIndexOfOffset = TokensBinaryEncoding.findIndexOfOffset;
 
 export class LineTokens implements ILineTokens {
 
@@ -707,7 +708,7 @@ export class LineTokens implements ILineTokens {
 	}
 
 	public toString(): string {
-		return LineTokensBinaryEncoding.inflateArr(this.map, this._tokens).toString();
+		return TokensBinaryEncoding.inflateArr(this.map, this._tokens).toString();
 	}
 
 	public getBinaryEncodedTokensMap(): ITokensInflatorMap {

src/vs/editor/common/model/textModelWithTokens.ts

@@ -23,6 +23,7 @@ import {ILineContext, ILineTokens, IMode, IModeTransition, IState} from 'vs/edit
 import {NullMode, NullState, nullTokenize} from 'vs/editor/common/modes/nullMode';
 import {ignoreBracketsInToken} from 'vs/editor/common/modes/supports';
 import {BracketsUtils} from 'vs/editor/common/modes/supports/richEditBrackets';
+import * as TokensBinaryEncoding from 'vs/editor/common/model/tokensBinaryEncoding';
 
 export class TokensInflatorMap implements editorCommon.ITokensInflatorMap {
 
@@ -1252,5 +1253,5 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 	}
 }
 
-var getType = editorCommon.LineTokensBinaryEncoding.getType;
-var getStartIndex = editorCommon.LineTokensBinaryEncoding.getStartIndex;
+var getType = TokensBinaryEncoding.getType;
+var getStartIndex = TokensBinaryEncoding.getStartIndex;

src/vs/editor/common/model/tokenIterator.ts

@@ -5,9 +5,10 @@
 'use strict';
 
 import * as editorCommon from 'vs/editor/common/editorCommon';
+import * as TokensBinaryEncoding from 'vs/editor/common/model/tokensBinaryEncoding';
 
-var getStartIndex = editorCommon.LineTokensBinaryEncoding.getStartIndex;
-var inflate = editorCommon.LineTokensBinaryEncoding.inflate;
+var getStartIndex = TokensBinaryEncoding.getStartIndex;
+var inflate = TokensBinaryEncoding.inflate;
 
 export class TokenIterator implements editorCommon.ITokenIterator {
 

src/vs/editor/common/model/tokensBinaryEncoding.ts

@@ -6,36 +6,19 @@
 
 import {onUnexpectedError} from 'vs/base/common/errors';
 import * as strings from 'vs/base/common/strings';
-import {ITokensInflatorMap} from 'vs/editor/common/editorCommon';
+import {ITokensInflatorMap, LineToken} from 'vs/editor/common/editorCommon';
 import {IToken} from 'vs/editor/common/modes';
 
-class InflatedToken implements IToken {
-	startIndex:number;
-	type:string;
-
-	constructor(startIndex:number, type:string) {
-		this.startIndex = startIndex;
-		this.type = type;
-	}
-
-	public toString(): string {
-		return '{ ' + this.startIndex + ', \'' + this.type + '\'}';
-	}
-}
-
 export var START_INDEX_MASK = 0xffffffff;
 export var TYPE_MASK = 0xffff;
 export var START_INDEX_OFFSET = 1;
 export var TYPE_OFFSET = Math.pow(2, 32);
 
-var DEFAULT_TOKEN = {
-	startIndex: 0,
-	type: ''
-};
-var INFLATED_TOKENS_EMPTY_TEXT = <IToken[]>[];
-var DEFLATED_TOKENS_EMPTY_TEXT = <number[]>[];
-var INFLATED_TOKENS_NON_EMPTY_TEXT = <IToken[]>[DEFAULT_TOKEN];
-var DEFLATED_TOKENS_NON_EMPTY_TEXT = <number[]>[0];
+var DEFAULT_TOKEN = new LineToken(0, '');
+var INFLATED_TOKENS_EMPTY_TEXT:LineToken[] = [];
+var DEFLATED_TOKENS_EMPTY_TEXT:number[] = [];
+var INFLATED_TOKENS_NON_EMPTY_TEXT:LineToken[] = [DEFAULT_TOKEN];
+var DEFLATED_TOKENS_NON_EMPTY_TEXT:number[] = [0];
 
 export function deflateArr(map:ITokensInflatorMap, tokens:IToken[]): number[] {
 	if (tokens.length === 0) {
@@ -105,7 +88,7 @@ export function inflate(map:ITokensInflatorMap, binaryEncodedToken:number): ITok
 	var startIndex = (binaryEncodedToken / START_INDEX_OFFSET) & START_INDEX_MASK;
 	var deflatedType = (binaryEncodedToken / TYPE_OFFSET) & TYPE_MASK;
 
-	return new InflatedToken(startIndex, map._inflate[deflatedType]);
+	return new LineToken(startIndex, map._inflate[deflatedType]);
 }
 
 export function getStartIndex(binaryEncodedToken:number): number {
@@ -120,7 +103,7 @@ export function getType(map:ITokensInflatorMap, binaryEncodedToken:number): stri
 	return map._inflate[deflatedType];
 }
 
-export function inflateArr(map:ITokensInflatorMap, binaryEncodedTokens:number[]): IToken[] {
+export function inflateArr(map:ITokensInflatorMap, binaryEncodedTokens:number[]): LineToken[] {
 	if (binaryEncodedTokens.length === 0) {
 		return INFLATED_TOKENS_EMPTY_TEXT;
 	}
@@ -128,7 +111,7 @@ export function inflateArr(map:ITokensInflatorMap, binaryEncodedTokens:number[])
 		return INFLATED_TOKENS_NON_EMPTY_TEXT;
 	}
 
-	var result: IToken[] = new Array(binaryEncodedTokens.length),
+	var result: LineToken[] = new Array(binaryEncodedTokens.length),
 		i:number,
 		len:number,
 		deflated:number,
@@ -142,7 +125,7 @@ export function inflateArr(map:ITokensInflatorMap, binaryEncodedTokens:number[])
 		startIndex = (deflated / START_INDEX_OFFSET) & START_INDEX_MASK;
 		deflatedType = (deflated / TYPE_OFFSET) & TYPE_MASK;
 
-		result[i] = new InflatedToken(startIndex, inflateMap[deflatedType]);
+		result[i] = new LineToken(startIndex, inflateMap[deflatedType]);
 	}
 
 	return result;
@@ -152,7 +135,7 @@ export function findIndexOfOffset(binaryEncodedTokens:number[], offset:number):
 	return findIndexInSegmentsArray(binaryEncodedTokens, offset);
 }
 
-export function sliceAndInflate(map:ITokensInflatorMap, binaryEncodedTokens:number[], startOffset:number, endOffset:number, deltaStartIndex:number): IToken[] {
+export function sliceAndInflate(map:ITokensInflatorMap, binaryEncodedTokens:number[], startOffset:number, endOffset:number, deltaStartIndex:number): LineToken[] {
 	if (binaryEncodedTokens.length === 0) {
 		return INFLATED_TOKENS_EMPTY_TEXT;
 	}
@@ -167,13 +150,13 @@ export function sliceAndInflate(map:ITokensInflatorMap, binaryEncodedTokens:numb
 		originalStartIndex:number,
 		newStartIndex:number,
 		deflatedType:number,
-		result: IToken[] = [],
+		result: LineToken[] = [],
 		inflateMap = map._inflate;
 
 	originalToken = binaryEncodedTokens[startIndex];
 	deflatedType = (originalToken / TYPE_OFFSET) & TYPE_MASK;
 	newStartIndex = 0;
-	result.push(new InflatedToken(newStartIndex, inflateMap[deflatedType]));
+	result.push(new LineToken(newStartIndex, inflateMap[deflatedType]));
 
 	for (i = startIndex + 1, len = binaryEncodedTokens.length; i < len; i++) {
 		originalToken = binaryEncodedTokens[i];
@@ -185,7 +168,7 @@ export function sliceAndInflate(map:ITokensInflatorMap, binaryEncodedTokens:numb
 
 		deflatedType = (originalToken / TYPE_OFFSET) & TYPE_MASK;
 		newStartIndex = originalStartIndex - startOffset + deltaStartIndex;
-		result.push(new InflatedToken(newStartIndex, inflateMap[deflatedType]));
+		result.push(new LineToken(newStartIndex, inflateMap[deflatedType]));
 	}
 
 	return result;

src/vs/editor/common/viewLayout/viewLineParts.ts

@@ -6,12 +6,12 @@
 
 import * as strings from 'vs/base/common/strings';
 import {Arrays} from 'vs/editor/common/core/arrays';
-import {ILineToken, IRange, IViewLineTokens} from 'vs/editor/common/editorCommon';
+import {LineToken, IRange, IViewLineTokens} from 'vs/editor/common/editorCommon';
 import {Range} from 'vs/editor/common/core/range';
 
 export interface ILineParts {
 
-	getParts(): ILineToken[];
+	getParts(): LineToken[];
 
 	equals(other:ILineParts): boolean;
 
@@ -38,7 +38,7 @@ export function createLineParts(lineNumber:number, minLineColumn:number, lineCon
 	}
 }
 
-function trimEmptyTrailingPart(parts: LinePart[], lineContent: string): LinePart[] {
+function trimEmptyTrailingPart(parts: LineToken[], lineContent: string): LineToken[] {
 	if (parts.length <= 1) {
 		return parts;
 	}
@@ -133,7 +133,7 @@ function insertWhitespace(lineNumber:number, lineContent: string, fauxIndentLeng
 export class FastViewLineParts implements ILineParts {
 
 	private lineTokens: IViewLineTokens;
-	private parts: LinePart[];
+	private parts: LineToken[];
 
 	constructor(lineTokens:IViewLineTokens, lineContent:string) {
 		this.lineTokens = lineTokens;
@@ -141,7 +141,7 @@ export class FastViewLineParts implements ILineParts {
 		this.parts = trimEmptyTrailingPart(this.parts, lineContent);
 	}
 
-	public getParts(): ILineToken[]{
+	public getParts(): LineToken[]{
 		return this.parts;
 	}
 
@@ -161,7 +161,7 @@ export class FastViewLineParts implements ILineParts {
 
 export class ViewLineParts implements ILineParts {
 
-	private parts:LinePart[];
+	private parts:LineToken[];
 	private lastPartIndex:number;
 	private lastEndOffset:number;
 
@@ -177,7 +177,7 @@ export class ViewLineParts implements ILineParts {
 			currentTokenEndOffset:number,
 			currentTokenClassName:string;
 
-		var parts:LinePart[] = [];
+		var parts:LineToken[] = [];
 
 		for (var i = 0, len = actualLineTokens.length; i < len; i++) {
 			nextStartOffset = actualLineTokens[i].startIndex;
@@ -187,11 +187,11 @@ export class ViewLineParts implements ILineParts {
 			while (lineDecorationsIndex < lineDecorationsLength && lineDecorations[lineDecorationsIndex].startOffset < currentTokenEndOffset) {
 				if (lineDecorations[lineDecorationsIndex].startOffset > nextStartOffset) {
 					// the first decorations starts after the token
-					parts.push(new LinePart(nextStartOffset, currentTokenClassName));
+					parts.push(new LineToken(nextStartOffset, currentTokenClassName));
 					nextStartOffset = lineDecorations[lineDecorationsIndex].startOffset;
 				}
 
-				parts.push(new LinePart(nextStartOffset, currentTokenClassName + ' ' + lineDecorations[lineDecorationsIndex].className));
+				parts.push(new LineToken(nextStartOffset, currentTokenClassName + ' ' + lineDecorations[lineDecorationsIndex].className));
 
 				if (lineDecorations[lineDecorationsIndex].endOffset >= currentTokenEndOffset) {
 					// this decoration goes on to the next token
@@ -205,7 +205,7 @@ export class ViewLineParts implements ILineParts {
 			}
 
 			if (nextStartOffset < currentTokenEndOffset) {
-				parts.push(new LinePart(nextStartOffset, currentTokenClassName));
+				parts.push(new LineToken(nextStartOffset, currentTokenClassName));
 			}
 		}
 
@@ -214,7 +214,7 @@ export class ViewLineParts implements ILineParts {
 		this.lastEndOffset = currentTokenEndOffset;
 	}
 
-	public getParts(): ILineToken[] {
+	public getParts(): LineToken[] {
 		return this.parts;
 	}
 
@@ -245,16 +245,6 @@ export class ViewLineParts implements ILineParts {
 	}
 }
 
-class LinePart implements ILineToken {
-	startIndex:number;
-	type:string;
-
-	constructor(startIndex:number, type:string) {
-		this.startIndex = startIndex;
-		this.type = type;
-	}
-}
-
 export class DecorationSegment {
 	startOffset:number;
 	endOffset:number;

src/vs/editor/common/viewLayout/viewLineRenderer.ts

@@ -4,14 +4,30 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
-import {ILineToken} from 'vs/editor/common/editorCommon';
+import {LineToken} from 'vs/editor/common/editorCommon';
+
+export class RenderLineInput {
+	public _renderLineInputTrait: void;
 
-export interface IRenderLineInput {
 	lineContent: string;
 	tabSize: number;
 	stopRenderingLineAfter: number;
 	renderWhitespace: boolean;
-	parts: ILineToken[];
+	parts: LineToken[];
+
+	constructor(
+		lineContent: string,
+		tabSize: number,
+		stopRenderingLineAfter: number,
+		renderWhitespace: boolean,
+		parts: LineToken[]
+	) {
+		this.lineContent = lineContent;
+		this.tabSize = tabSize;
+		this.stopRenderingLineAfter = stopRenderingLineAfter;
+		this.renderWhitespace = renderWhitespace;
+		this.parts = parts;
+	}
 }
 
 export interface IRenderLineOutput {
@@ -29,7 +45,7 @@ const _carriageReturn = '\r'.charCodeAt(0);
 const _lineSeparator = '\u2028'.charCodeAt(0); //http://www.fileformat.info/info/unicode/char/2028/index.htm
 const _bom = 65279;
 
-export function renderLine(input:IRenderLineInput): IRenderLineOutput {
+export function renderLine(input:RenderLineInput): IRenderLineOutput {
 	const lineText = input.lineContent;
 	const lineTextLength = lineText.length;
 	const tabSize = input.tabSize;
@@ -50,6 +66,19 @@ export function renderLine(input:IRenderLineInput): IRenderLineOutput {
 		throw new Error('Cannot render non empty line without line parts!');
 	}
 
+	return renderLineActual(lineText, lineTextLength, tabSize, actualLineParts, renderWhitespace, charBreakIndex);
+}
+
+const WHITESPACE_TOKEN_TEST = /\bwhitespace\b/;
+function isWhitespace(type:string): boolean {
+	return WHITESPACE_TOKEN_TEST.test(type);
+}
+
+function renderLineActual(lineText:string, lineTextLength:number, tabSize:number, actualLineParts:LineToken[], renderWhitespace:boolean, charBreakIndex:number): IRenderLineOutput {
+	lineTextLength = +lineTextLength;
+	tabSize = +tabSize;
+	charBreakIndex = +charBreakIndex;
+
 	let charIndex = 0;
 	let out: string[] = [];
 	let charOffsetInPartArr: number[] = [];
@@ -60,19 +89,19 @@ export function renderLine(input:IRenderLineInput): IRenderLineOutput {
 	for (let partIndex = 0, partIndexLen = actualLineParts.length; partIndex < partIndexLen; partIndex++) {
 		let part = actualLineParts[partIndex];
 
-		out.push('<span class="');
-		out.push('token ');
-		out.push(part.type.replace(/[^a-z0-9\-]/gi, ' '));
+		out.push('<span class="token ');
+		out.push(part.type);
 		out.push('">');
 
 		let partRendersWhitespace = false;
 		if (renderWhitespace) {
-			partRendersWhitespace = (/\bwhitespace\b/.test(part.type));
+			partRendersWhitespace = isWhitespace(part.type);
 		}
 
 		let toCharIndex = lineTextLength;
 		if (partIndex + 1 < partIndexLen) {
-			toCharIndex = Math.min(lineTextLength, actualLineParts[partIndex + 1].startIndex);
+			let nextPart = actualLineParts[partIndex + 1];
+			toCharIndex = Math.min(lineTextLength, nextPart.startIndex);
 		}
 
 		charOffsetInPart = 0;

src/vs/editor/common/viewModel/filteredLineTokens.ts

@@ -5,11 +5,12 @@
 'use strict';
 
 import {Arrays} from 'vs/editor/common/core/arrays';
-import {ILineToken, ILineTokens, IViewLineTokens, LineTokensBinaryEncoding} from 'vs/editor/common/editorCommon';
+import {LineToken, ILineTokens, IViewLineTokens} from 'vs/editor/common/editorCommon';
+import * as TokensBinaryEncoding from 'vs/editor/common/model/tokensBinaryEncoding';
 
 export class FilteredLineTokens implements IViewLineTokens {
 
-	private inflatedTokens: ILineToken[];
+	private inflatedTokens: LineToken[];
 
 	private _original:ILineTokens;
 	private _startOffset:number;
@@ -25,10 +26,10 @@ export class FilteredLineTokens implements IViewLineTokens {
 		this._endOffset = endOffset;
 		this._deltaStartIndex = deltaStartIndex;
 
-		this.inflatedTokens = LineTokensBinaryEncoding.sliceAndInflate(original.getBinaryEncodedTokensMap(), original.getBinaryEncodedTokens(), startOffset, endOffset, deltaStartIndex);
+		this.inflatedTokens = TokensBinaryEncoding.sliceAndInflate(original.getBinaryEncodedTokensMap(), original.getBinaryEncodedTokens(), startOffset, endOffset, deltaStartIndex);
 	}
 
-	public getTokens(): ILineToken[]{
+	public getTokens(): LineToken[]{
 		return this.inflatedTokens;
 	}
 
@@ -72,8 +73,8 @@ export class IdentityFilteredLineTokens implements IViewLineTokens {
 		this._textLength = textLength;
 	}
 
-	public getTokens(): ILineToken[] {
-		return LineTokensBinaryEncoding.inflateArr(this._original.getBinaryEncodedTokensMap(), this._original.getBinaryEncodedTokens());
+	public getTokens(): LineToken[] {
+		return TokensBinaryEncoding.inflateArr(this._original.getBinaryEncodedTokensMap(), this._original.getBinaryEncodedTokens());
 	}
 
 	public getFauxIndentLength(): number {

src/vs/editor/test/common/model/model.line.test.ts

@@ -5,14 +5,15 @@
 'use strict';
 
 import * as assert from 'assert';
-import {ILineTokens, LineTokensBinaryEncoding} from 'vs/editor/common/editorCommon';
+import {ILineTokens} from 'vs/editor/common/editorCommon';
 import * as modelLine from 'vs/editor/common/model/modelLine';
 import {LineMarker} from 'vs/editor/common/model/textModelWithMarkers';
 import {TokensInflatorMap} from 'vs/editor/common/model/textModelWithTokens';
 import {IToken} from 'vs/editor/common/modes';
+import * as TokensBinaryEncoding from 'vs/editor/common/model/tokensBinaryEncoding';
 
 function assertLineTokens(actual:ILineTokens, expected:IToken[]): void {
-	var inflatedActual = LineTokensBinaryEncoding.inflateArr(actual.getBinaryEncodedTokensMap(), actual.getBinaryEncodedTokens());
+	var inflatedActual = TokensBinaryEncoding.inflateArr(actual.getBinaryEncodedTokensMap(), actual.getBinaryEncodedTokens());
 	assert.deepEqual(inflatedActual, expected, 'Line tokens are equal');
 }
 

src/vs/editor/test/common/viewLayout/viewLineRenderer.test.ts

@@ -5,26 +5,23 @@
 'use strict';
 
 import * as assert from 'assert';
-import {ILineToken} from 'vs/editor/common/editorCommon';
-import {renderLine} from 'vs/editor/common/viewLayout/viewLineRenderer';
+import {LineToken} from 'vs/editor/common/editorCommon';
+import {renderLine, RenderLineInput} from 'vs/editor/common/viewLayout/viewLineRenderer';
 
 suite('viewLineRenderer.renderLine', () => {
 
-	function createPart(startIndex: number, type:string): ILineToken {
-		return {
-			startIndex: startIndex,
-			type: type
-		};
+	function createPart(startIndex: number, type:string): LineToken {
+		return new LineToken(startIndex, type);
 	}
 
 	function assertCharacterReplacement(lineContent:string, tabSize:number, expected:string, expectedCharOffsetInPart: number[]): void {
-		let _actual = renderLine({
-			lineContent: lineContent,
-			tabSize: tabSize,
-			stopRenderingLineAfter: -1,
-			renderWhitespace: false,
-			parts: [createPart(0, '')]
-		});
+		let _actual = renderLine(new RenderLineInput(
+			lineContent,
+			tabSize,
+			-1,
+			false,
+			[createPart(0, '')]
+		));
 
 		assert.equal(_actual.output.join(''), '<span><span class="token ">' + expected + '</span></span>');
 		assert.deepEqual(_actual.charOffsetInPart, expectedCharOffsetInPart);
@@ -57,14 +54,14 @@ suite('viewLineRenderer.renderLine', () => {
 		assertCharacterReplacement('xxxx\t', 4, 'xxxx&nbsp;&nbsp;&nbsp;&nbsp;', [0, 1, 2, 3, 4, 8]);
 	});
 
-	function assertParts(lineContent:string, tabSize:number, parts: ILineToken[], expected:string, expectedCharOffsetInPart:number[]): void {
-		let _actual = renderLine({
-			lineContent: lineContent,
-			tabSize: tabSize,
-			stopRenderingLineAfter: -1,
-			renderWhitespace: false,
-			parts: parts
-		});
+	function assertParts(lineContent:string, tabSize:number, parts: LineToken[], expected:string, expectedCharOffsetInPart:number[]): void {
+		let _actual = renderLine(new RenderLineInput(
+			lineContent,
+			tabSize,
+			-1,
+			false,
+			parts
+		));
 
 		assert.equal(_actual.output.join(''), '<span>' + expected + '</span>');
 		assert.deepEqual(_actual.charOffsetInPart, expectedCharOffsetInPart);
@@ -87,9 +84,12 @@ suite('viewLineRenderer.renderLine', () => {
 	});
 
 	test('overflow', () => {
-		let _actual = renderLine({
-			lineContent: 'Hello world!',
-			parts: [
+		let _actual = renderLine(new RenderLineInput(
+			'Hello world!',
+			4,
+			6,
+			true,
+			[
 				createPart( 0,  '0'),
 				createPart( 1,  '1'),
 				createPart( 2,  '2'),
@@ -102,11 +102,8 @@ suite('viewLineRenderer.renderLine', () => {
 				createPart( 9,  '9'),
 				createPart(10, '10'),
 				createPart(11, '11'),
-			],
-			tabSize: 4,
-			stopRenderingLineAfter: 6,
-			renderWhitespace: true,
-		});
+			]
+		));
 
 		let expectedOutput = [
 			'<span class="token 0">H</span>',
@@ -174,13 +171,13 @@ suite('viewLineRenderer.renderLine', () => {
 		];
 		let expectedOffsets = expectedOffsetsArr.reduce((prev, curr) => prev.concat(curr), []);
 
-		let _actual = renderLine({
-			lineContent: lineText,
-			tabSize: 4,
-			stopRenderingLineAfter: -1,
-			renderWhitespace: true,
-			parts: lineParts
-		});
+		let _actual = renderLine(new RenderLineInput(
+			lineText,
+			4,
+			-1,
+			true,
+			lineParts
+		));
 
 		assert.equal(_actual.output.join(''), '<span>' + expectedOutput + '</span>');
 		assert.deepEqual(_actual.charOffsetInPart, expectedOffsets);
@@ -227,13 +224,13 @@ suite('viewLineRenderer.renderLine', () => {
 		];
 		let expectedOffsets = expectedOffsetsArr.reduce((prev, curr) => prev.concat(curr), []);
 
-		let _actual = renderLine({
-			lineContent: lineText,
-			tabSize: 4,
-			stopRenderingLineAfter: -1,
-			renderWhitespace: false,
-			parts: lineParts
-		});
+		let _actual = renderLine(new RenderLineInput(
+			lineText,
+			4,
+			-1,
+			false,
+			lineParts
+		));
 
 		assert.equal(_actual.output.join(''), '<span>' + expectedOutput + '</span>');
 		assert.deepEqual(_actual.charOffsetInPart, expectedOffsets);
@@ -280,13 +277,13 @@ suite('viewLineRenderer.renderLine', () => {
 		];
 		let expectedOffsets = expectedOffsetsArr.reduce((prev, curr) => prev.concat(curr), []);
 
-		let _actual = renderLine({
-			lineContent: lineText,
-			tabSize: 4,
-			stopRenderingLineAfter: -1,
-			renderWhitespace: false,
-			parts: lineParts
-		});
+		let _actual = renderLine(new RenderLineInput(
+			lineText,
+			4,
+			-1,
+			false,
+			lineParts
+		));
 
 		assert.equal(_actual.output.join(''), '<span>' + expectedOutput + '</span>');
 		assert.deepEqual(_actual.charOffsetInPart, expectedOffsets);