Fixes Microsoft/monaco-editor#122: Clear all tokenization data when resetting tokenization state

src/vs/editor/common/model/modelLine.ts

@@ -160,6 +160,12 @@ export class ModelLine {
 
 	// --- BEGIN STATE
 
+	public resetTokenizationState(): void {
+		this._state = null;
+		this._modeTransitions = null;
+		this._lineTokens = null;
+	}
+
 	public setState(state: IState): void {
 		this._state = state;
 	}

src/vs/editor/common/model/textModelWithTokens.ts

@@ -354,19 +354,9 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 		this._scheduleRetokenizeNow.cancel();
 		this._clearTimers();
 		for (var i = 0; i < this._lines.length; i++) {
-			this._lines[i].setState(null);
+			this._lines[i].resetTokenizationState();
 		}
-		this._initializeTokenizationState();
-	}
-
-	private _clearTimers(): void {
-		if (this._revalidateTokensTimeout !== -1) {
-			clearTimeout(this._revalidateTokensTimeout);
-			this._revalidateTokensTimeout = -1;
-		}
-	}
 
-	private _initializeTokenizationState(): void {
 		// Initialize tokenization states
 		var initialState:IState = null;
 		if (this._mode.tokenizationSupport) {
@@ -389,6 +379,13 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 		this._beginBackgroundTokenization();
 	}
 
+	private _clearTimers(): void {
+		if (this._revalidateTokensTimeout !== -1) {
+			clearTimeout(this._revalidateTokensTimeout);
+			this._revalidateTokensTimeout = -1;
+		}
+	}
+
 	public getLineTokens(lineNumber:number, inaccurateTokensAcceptable:boolean = false): editorCommon.ILineTokens {
 		if (lineNumber < 1 || lineNumber > this.getLineCount()) {
 			throw new Error('Illegal value ' + lineNumber + ' for `lineNumber`');

src/vs/editor/test/common/model/textModelWithTokens.test.ts

+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import * as assert from 'assert';
+import {Model} from 'vs/editor/common/model/model';
+import {ViewLineToken} from 'vs/editor/common/core/viewLineToken';
+import {ITokenizationSupport} from 'vs/editor/common/modes';
+import {MockMode} from 'vs/editor/test/common/mocks/mockMode';
+
+suite('TextModelWithTokens', () => {
+
+	function assertViewLineTokens(model:Model, lineNumber:number, forceTokenization:boolean, expected:ViewLineToken[]): void {
+		let actual = model.getLineTokens(lineNumber, !forceTokenization).inflate();
+		assert.deepEqual(actual, expected);
+	}
+
+	test('Microsoft/monaco-editor#122: Unhandled Exception: TypeError: Unable to get property \'replace\' of undefined or null reference', () => {
+		let _tokenId = 0;
+		class IndicisiveMode extends MockMode {
+			public tokenizationSupport:ITokenizationSupport;
+
+			constructor() {
+				super();
+				this.tokenizationSupport = {
+					getInitialState: () => {
+						return null;
+					},
+					tokenize: (line, state, offsetDelta, stopAtOffset) => {
+						let myId = ++_tokenId;
+						return {
+							tokens: [{ startIndex: 0, type: 'custom.'+myId }],
+							actualStopOffset: line.length,
+							endState: null,
+							modeTransitions: [],
+							retokenize: null
+						};
+					}
+				};
+			}
+		}
+		let model = Model.createFromString('A model with\ntwo lines');
+
+		assertViewLineTokens(model, 1, true, [new ViewLineToken(0, '')]);
+		assertViewLineTokens(model, 2, true, [new ViewLineToken(0, '')]);
+
+		model.setMode(new IndicisiveMode());
+
+		assertViewLineTokens(model, 1, true, [new ViewLineToken(0, 'custom.1')]);
+		assertViewLineTokens(model, 2, true, [new ViewLineToken(0, 'custom.2')]);
+
+		model.setMode(new IndicisiveMode());
+
+		assertViewLineTokens(model, 1, false, [new ViewLineToken(0, '')]);
+		assertViewLineTokens(model, 2, false, [new ViewLineToken(0, '')]);
+
+		model.dispose();
+	});
+
+});