Add & adopt forceTokenization

src/vs/editor/browser/standalone/colorizer.ts

@@ -113,7 +113,8 @@ export class Colorizer {
 
 	public static colorizeModelLine(model: IModel, lineNumber: number, tabSize: number = 4): string {
 		let content = model.getLineContent(lineNumber);
-		let tokens = model.getLineTokens(lineNumber, false);
+		model.forceTokenization(lineNumber);
+		let tokens = model.getLineTokens(lineNumber);
 		let inflatedTokens = tokens.inflate();
 		return this.colorizeLine(content, model.mightContainRTL(), inflatedTokens, tabSize);
 	}

src/vs/editor/browser/widget/codeEditorWidget.ts

@@ -145,7 +145,8 @@ export abstract class CodeEditorWidget extends CommonCodeEditor implements edito
 			return '';
 		}
 		let content = model.getLineContent(lineNumber);
-		let tokens = model.getLineTokens(lineNumber, false);
+		model.forceTokenization(lineNumber);
+		let tokens = model.getLineTokens(lineNumber);
 		let inflatedTokens = tokens.inflate();
 		let tabSize = model.getOptions().tabSize;
 		return Colorizer.colorizeLine(content, model.mightContainRTL(), inflatedTokens, tabSize);

src/vs/editor/common/controller/cursorTypeOperations.ts

@@ -344,7 +344,8 @@ export class TypeOperations {
 			}
 		}
 
-		let lineTokens = model.getLineTokens(position.lineNumber, false);
+		model.forceTokenization(position.lineNumber);
+		let lineTokens = model.getLineTokens(position.lineNumber);
 
 		let shouldAutoClosePair = false;
 		try {
@@ -413,7 +414,8 @@ export class TypeOperations {
 		}
 
 		let position = cursor.position;
-		let lineTokens = model.getLineTokens(position.lineNumber, false);
+		model.forceTokenization(position.lineNumber);
+		let lineTokens = model.getLineTokens(position.lineNumber);
 
 		let electricAction: IElectricAction;
 		try {

src/vs/editor/common/editorCommon.ts

@@ -2011,12 +2011,17 @@ export interface IFoundBracket {
 export interface ITokenizedModel extends ITextModel {
 
 	/**
-	 * Tokenize if necessary and get the tokens for the line `lineNumber`.
-	 * @param lineNumber The line number
-	 * @param inaccurateTokensAcceptable Are inaccurate tokens acceptable? Defaults to false
+	 * Force tokenization information for `lineNumber` to be accurate.
+	 * @internal
+	 */
+	forceTokenization(lineNumber: number): void;
+
+	/**
+	 * Get the tokens for the line `lineNumber`.
+	 * The tokens might be inaccurate. Use `forceTokenization` to ensure accurate tokens.
 	 * @internal
 	 */
-	getLineTokens(lineNumber: number, inaccurateTokensAcceptable: boolean): LineTokens;
+	getLineTokens(lineNumber: number): LineTokens;
 
 	/**
 	 * Get the language associated with this model.

src/vs/editor/common/model/textModelWithTokens.ts

@@ -167,15 +167,19 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 		return result;
 	}
 
-	public getLineTokens(lineNumber: number, inaccurateTokensAcceptable: boolean): LineTokens {
+	public forceTokenization(lineNumber: number): void {
 		if (lineNumber < 1 || lineNumber > this.getLineCount()) {
 			throw new Error('Illegal value ' + lineNumber + ' for `lineNumber`');
 		}
 
-		if (!inaccurateTokensAcceptable) {
-			this._withModelTokensChangedEventBuilder((eventBuilder) => {
-				this._updateTokensUntilLine(eventBuilder, lineNumber, true);
-			});
+		this._withModelTokensChangedEventBuilder((eventBuilder) => {
+			this._updateTokensUntilLine(eventBuilder, lineNumber);
+		});
+	}
+
+	public getLineTokens(lineNumber: number): LineTokens {
+		if (lineNumber < 1 || lineNumber > this.getLineCount()) {
+			throw new Error('Illegal value ' + lineNumber + ' for `lineNumber`');
 		}
 
 		return this._getLineTokens(lineNumber);
@@ -225,7 +229,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 		let { lineNumber, column } = this.validatePosition({ lineNumber: _lineNumber, column: _column });
 
 		this._withModelTokensChangedEventBuilder((eventBuilder) => {
-			this._updateTokensUntilLine(eventBuilder, lineNumber, true);
+			this._updateTokensUntilLine(eventBuilder, lineNumber);
 		});
 
 		let lineTokens = this._getLineTokens(lineNumber);
@@ -311,7 +315,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 					}
 				}
 
-				this._updateTokensUntilLine(eventBuilder, lineNumber, false);
+				this._updateTokensUntilLine(eventBuilder, lineNumber);
 				tokenizedChars += currentCharsToTokenize;
 
 				// Skip the lines that got tokenized
@@ -326,7 +330,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 		});
 	}
 
-	private _updateTokensUntilLine(eventBuilder: ModelTokensChangedEventBuilder, lineNumber: number, emitEvents: boolean): void {
+	private _updateTokensUntilLine(eventBuilder: ModelTokensChangedEventBuilder, lineNumber: number): void {
 		if (!this._tokenizationSupport) {
 			this._invalidLineStartIndex = this._lines.length;
 			return;

src/vs/editor/common/model/tokenIterator.ts

@@ -82,7 +82,8 @@ export class TokenIterator implements editorCommon.ITokenIterator {
 
 		position = findClosestNonEmptyLine(model, position);
 		if (position) {
-			let lineTokens = this._model.getLineTokens(position.lineNumber, false);
+			this._model.forceTokenization(position.lineNumber);
+			let lineTokens = this._model.getLineTokens(position.lineNumber);
 			let currentToken = lineTokens.findTokenAtOffset(position.column - 1);
 			if (currentToken) {
 				this._prev = this._next = new TokenInfo(currentToken, position.lineNumber);
@@ -99,7 +100,8 @@ export class TokenIterator implements editorCommon.ITokenIterator {
 		let next = this._next._actual.next();
 		while (!next && lineNumber < this._lineCount) {
 			lineNumber++;
-			let currentLineTokens = this._model.getLineTokens(lineNumber, false);
+			this._model.forceTokenization(lineNumber);
+			let currentLineTokens = this._model.getLineTokens(lineNumber);
 			next = currentLineTokens.firstToken();
 		}
 
@@ -120,7 +122,8 @@ export class TokenIterator implements editorCommon.ITokenIterator {
 		let prev = this._prev._actual.prev();
 		while (!prev && lineNumber > 1) {
 			lineNumber--;
-			let currentLineTokens = this._model.getLineTokens(lineNumber, false);
+			this._model.forceTokenization(lineNumber);
+			let currentLineTokens = this._model.getLineTokens(lineNumber);
 			prev = currentLineTokens.lastToken();
 		}
 

src/vs/editor/common/modes/languageConfigurationRegistry.ts

@@ -399,7 +399,8 @@ export class LanguageConfigurationRegistryImpl {
 	}
 
 	private getScopedLineTokens(model: ITokenizedModel, lineNumber: number) {
-		let lineTokens = model.getLineTokens(lineNumber, false);
+		model.forceTokenization(lineNumber);
+		let lineTokens = model.getLineTokens(lineNumber);
 		let column = model.getLineMaxColumn(lineNumber);
 		let scopedLineTokens = createScopedLineTokens(lineTokens, column - 1);
 		return scopedLineTokens;

src/vs/editor/common/viewModel/splitLinesCollection.ts

@@ -35,7 +35,7 @@ export interface ILineMapperFactory {
 }
 
 export interface IModel {
-	getLineTokens(lineNumber: number, inaccurateTokensAcceptable: boolean): LineTokens;
+	getLineTokens(lineNumber: number): LineTokens;
 	getLineContent(lineNumber: number): string;
 	getLineMinColumn(lineNumber: number): number;
 	getLineMaxColumn(lineNumber: number): number;
@@ -90,7 +90,7 @@ class VisibleIdentitySplitLine implements ISplitLine {
 	}
 
 	public getViewLineData(model: IModel, modelLineNumber: number, outputLineIndex: number): ViewLineData {
-		let lineTokens = model.getLineTokens(modelLineNumber, true);
+		let lineTokens = model.getLineTokens(modelLineNumber);
 		let lineContent = lineTokens.getLineContent();
 		return new ViewLineData(
 			lineContent,
@@ -263,7 +263,7 @@ export class SplitLine implements ISplitLine {
 		if (outputLineIndex > 0) {
 			deltaStartIndex = this.wrappedIndentLength;
 		}
-		let lineTokens = model.getLineTokens(modelLineNumber, true);
+		let lineTokens = model.getLineTokens(modelLineNumber);
 
 		return new ViewLineData(
 			lineContent,

src/vs/editor/common/viewModel/viewModelImpl.ts

@@ -667,7 +667,7 @@ export class ViewModel implements IViewModel {
 		let result = '';
 
 		for (let lineNumber = startLineNumber; lineNumber <= endLineNumber; lineNumber++) {
-			const lineTokens = this.model.getLineTokens(lineNumber, true);
+			const lineTokens = this.model.getLineTokens(lineNumber);
 			const lineContent = lineTokens.getLineContent();
 			const startOffset = (lineNumber === startLineNumber ? startColumn - 1 : 0);
 			const endOffset = (lineNumber === endLineNumber ? endColumn - 1 : lineContent.length);

src/vs/editor/contrib/smartSelect/common/tokenTree.ts

@@ -165,7 +165,8 @@ class ModelRawTokenScanner {
 		while (!this._next && this._lineNumber < this._lineCount) {
 			this._lineNumber++;
 			this._lineText = this._model.getLineContent(this._lineNumber);
-			let currentLineTokens = this._model.getLineTokens(this._lineNumber, false);
+			this._model.forceTokenization(this._lineNumber);
+			let currentLineTokens = this._model.getLineTokens(this._lineNumber);
 			this._next = currentLineTokens.firstToken();
 		}
 	}

src/vs/editor/test/common/model/model.modes.test.ts

@@ -59,98 +59,98 @@ suite('Editor Model - Model Modes 1', () => {
 	});
 
 	test('model calls syntax highlighter 1', () => {
-		thisModel.getLineTokens(1, false);
+		thisModel.forceTokenization(1);
 		checkAndClear(['1']);
 	});
 
 	test('model calls syntax highlighter 2', () => {
-		thisModel.getLineTokens(2, false);
+		thisModel.forceTokenization(2);
 		checkAndClear(['1', '2']);
 
-		thisModel.getLineTokens(2, false);
+		thisModel.forceTokenization(2);
 		checkAndClear([]);
 	});
 
 	test('model caches states', () => {
-		thisModel.getLineTokens(1, false);
+		thisModel.forceTokenization(1);
 		checkAndClear(['1']);
 
-		thisModel.getLineTokens(2, false);
+		thisModel.forceTokenization(2);
 		checkAndClear(['2']);
 
-		thisModel.getLineTokens(3, false);
+		thisModel.forceTokenization(3);
 		checkAndClear(['3']);
 
-		thisModel.getLineTokens(4, false);
+		thisModel.forceTokenization(4);
 		checkAndClear(['4']);
 
-		thisModel.getLineTokens(5, false);
+		thisModel.forceTokenization(5);
 		checkAndClear(['5']);
 
-		thisModel.getLineTokens(5, false);
+		thisModel.forceTokenization(5);
 		checkAndClear([]);
 	});
 
 	test('model invalidates states for one line insert', () => {
-		thisModel.getLineTokens(5, false);
+		thisModel.forceTokenization(5);
 		checkAndClear(['1', '2', '3', '4', '5']);
 
 		thisModel.applyEdits([EditOperation.insert(new Position(1, 1), '-')]);
-		thisModel.getLineTokens(5, false);
+		thisModel.forceTokenization(5);
 		checkAndClear(['-']);
 
-		thisModel.getLineTokens(5, false);
+		thisModel.forceTokenization(5);
 		checkAndClear([]);
 	});
 
 	test('model invalidates states for many lines insert', () => {
-		thisModel.getLineTokens(5, false);
+		thisModel.forceTokenization(5);
 		checkAndClear(['1', '2', '3', '4', '5']);
 
 		thisModel.applyEdits([EditOperation.insert(new Position(1, 1), '0\n-\n+')]);
 		assert.equal(thisModel.getLineCount(), 7);
-		thisModel.getLineTokens(7, false);
+		thisModel.forceTokenization(7);
 		checkAndClear(['0', '-', '+']);
 
-		thisModel.getLineTokens(7, false);
+		thisModel.forceTokenization(7);
 		checkAndClear([]);
 	});
 
 	test('model invalidates states for one new line', () => {
-		thisModel.getLineTokens(5, false);
+		thisModel.forceTokenization(5);
 		checkAndClear(['1', '2', '3', '4', '5']);
 
 		thisModel.applyEdits([EditOperation.insert(new Position(1, 2), '\n')]);
 		thisModel.applyEdits([EditOperation.insert(new Position(2, 1), 'a')]);
-		thisModel.getLineTokens(6, false);
+		thisModel.forceTokenization(6);
 		checkAndClear(['1', 'a']);
 	});
 
 	test('model invalidates states for one line delete', () => {
-		thisModel.getLineTokens(5, false);
+		thisModel.forceTokenization(5);
 		checkAndClear(['1', '2', '3', '4', '5']);
 
 		thisModel.applyEdits([EditOperation.insert(new Position(1, 2), '-')]);
-		thisModel.getLineTokens(5, false);
+		thisModel.forceTokenization(5);
 		checkAndClear(['1']);
 
 		thisModel.applyEdits([EditOperation.delete(new Range(1, 1, 1, 2))]);
-		thisModel.getLineTokens(5, false);
+		thisModel.forceTokenization(5);
 		checkAndClear(['-']);
 
-		thisModel.getLineTokens(5, false);
+		thisModel.forceTokenization(5);
 		checkAndClear([]);
 	});
 
 	test('model invalidates states for many lines delete', () => {
-		thisModel.getLineTokens(5, false);
+		thisModel.forceTokenization(5);
 		checkAndClear(['1', '2', '3', '4', '5']);
 
 		thisModel.applyEdits([EditOperation.delete(new Range(1, 1, 3, 1))]);
-		thisModel.getLineTokens(3, false);
+		thisModel.forceTokenization(3);
 		checkAndClear(['3']);
 
-		thisModel.getLineTokens(3, false);
+		thisModel.forceTokenization(3);
 		checkAndClear([]);
 	});
 });
@@ -227,17 +227,17 @@ suite('Editor Model - Model Modes 2', () => {
 	});
 
 	test('getTokensForInvalidLines one text insert', () => {
-		thisModel.getLineTokens(5, false);
+		thisModel.forceTokenization(5);
 		statesEqual(thisModel, ['', 'Line1', 'Line2', 'Line3', 'Line4', 'Line5']);
 		thisModel.applyEdits([EditOperation.insert(new Position(1, 6), '-')]);
 		invalidEqual(thisModel, [0]);
 		statesEqual(thisModel, ['', 'Line1', 'Line2', 'Line3', 'Line4', 'Line5']);
-		thisModel.getLineTokens(5, false);
+		thisModel.forceTokenization(5);
 		statesEqual(thisModel, ['', 'Line1-', 'Line2', 'Line3', 'Line4', 'Line5']);
 	});
 
 	test('getTokensForInvalidLines two text insert', () => {
-		thisModel.getLineTokens(5, false);
+		thisModel.forceTokenization(5);
 		statesEqual(thisModel, ['', 'Line1', 'Line2', 'Line3', 'Line4', 'Line5']);
 		thisModel.applyEdits([
 			EditOperation.insert(new Position(1, 6), '-'),
@@ -245,46 +245,46 @@ suite('Editor Model - Model Modes 2', () => {
 		]);
 
 		invalidEqual(thisModel, [0, 2]);
-		thisModel.getLineTokens(5, false);
+		thisModel.forceTokenization(5);
 		statesEqual(thisModel, ['', 'Line1-', 'Line2', 'Line3-', 'Line4', 'Line5']);
 	});
 
 	test('getTokensForInvalidLines one multi-line text insert, one small text insert', () => {
-		thisModel.getLineTokens(5, false);
+		thisModel.forceTokenization(5);
 		statesEqual(thisModel, ['', 'Line1', 'Line2', 'Line3', 'Line4', 'Line5']);
 		thisModel.applyEdits([EditOperation.insert(new Position(1, 6), '\nNew line\nAnother new line')]);
 		thisModel.applyEdits([EditOperation.insert(new Position(5, 6), '-')]);
 		invalidEqual(thisModel, [0, 4]);
-		thisModel.getLineTokens(7, false);
+		thisModel.forceTokenization(7);
 		statesEqual(thisModel, ['', 'Line1', 'New line', 'Another new line', 'Line2', 'Line3-', 'Line4', 'Line5']);
 	});
 
 	test('getTokensForInvalidLines one delete text', () => {
-		thisModel.getLineTokens(5, false);
+		thisModel.forceTokenization(5);
 		statesEqual(thisModel, ['', 'Line1', 'Line2', 'Line3', 'Line4', 'Line5']);
 		thisModel.applyEdits([EditOperation.delete(new Range(1, 1, 1, 5))]);
 		invalidEqual(thisModel, [0]);
-		thisModel.getLineTokens(5, false);
+		thisModel.forceTokenization(5);
 		statesEqual(thisModel, ['', '1', 'Line2', 'Line3', 'Line4', 'Line5']);
 	});
 
 	test('getTokensForInvalidLines one line delete text', () => {
-		thisModel.getLineTokens(5, false);
+		thisModel.forceTokenization(5);
 		statesEqual(thisModel, ['', 'Line1', 'Line2', 'Line3', 'Line4', 'Line5']);
 		thisModel.applyEdits([EditOperation.delete(new Range(1, 1, 2, 1))]);
 		invalidEqual(thisModel, [0]);
 		statesEqual(thisModel, ['', 'Line2', 'Line3', 'Line4', 'Line5']);
-		thisModel.getLineTokens(4, false);
+		thisModel.forceTokenization(4);
 		statesEqual(thisModel, ['', 'Line2', 'Line3', 'Line4', 'Line5']);
 	});
 
 	test('getTokensForInvalidLines multiple lines delete text', () => {
-		thisModel.getLineTokens(5, false);
+		thisModel.forceTokenization(5);
 		statesEqual(thisModel, ['', 'Line1', 'Line2', 'Line3', 'Line4', 'Line5']);
 		thisModel.applyEdits([EditOperation.delete(new Range(1, 1, 3, 3))]);
 		invalidEqual(thisModel, [0]);
 		statesEqual(thisModel, ['', 'Line3', 'Line4', 'Line5']);
-		thisModel.getLineTokens(3, false);
+		thisModel.forceTokenization(3);
 		statesEqual(thisModel, ['', 'ne3', 'Line4', 'Line5']);
 	});
 });

src/vs/editor/test/common/model/textModelWithTokens.test.ts

@@ -266,7 +266,10 @@ suite('TextModelWithTokens regression tests', () => {
 
 	test('Microsoft/monaco-editor#122: Unhandled Exception: TypeError: Unable to get property \'replace\' of undefined or null reference', () => {
 		function assertViewLineTokens(model: Model, lineNumber: number, forceTokenization: boolean, expected: ViewLineToken[]): void {
-			let actual = model.getLineTokens(lineNumber, !forceTokenization).inflate();
+			if (forceTokenization) {
+				model.forceTokenization(lineNumber);
+			}
+			let actual = model.getLineTokens(lineNumber).inflate();
 			let decode = (token: ViewLineToken) => {
 				return {
 					endIndex: token.endIndex,

src/vs/editor/test/common/viewModel/splitLinesCollection.test.ts

@@ -352,7 +352,7 @@ suite('SplitLinesCollection', () => {
 		languageRegistration = modes.TokenizationRegistry.register(LANGUAGE_ID, tokenizationSupport);
 		model = Model.createFromString(_text.join('\n'), undefined, new modes.LanguageIdentifier(LANGUAGE_ID, 0));
 		// force tokenization
-		model.getLineTokens(model.getLineCount(), false);
+		model.forceTokenization(model.getLineCount());
 	});
 
 	teardown(() => {
@@ -777,7 +777,7 @@ function createLineMapping(breakingLengths: number[], wrappedLinesPrefix: string
 
 function createModel(text: string): IModel {
 	return {
-		getLineTokens: (lineNumber: number, inaccurateTokensAcceptable?: boolean) => {
+		getLineTokens: (lineNumber: number) => {
 			return null;
 		},
 		getLineContent: (lineNumber: number) => {

src/vs/workbench/parts/debug/electron-browser/debugEditorContribution.ts

@@ -514,7 +514,8 @@ export class DebugEditorContribution implements IDebugEditorContribution {
 					continue;
 				}
 
-				const lineTokens = model.getLineTokens(lineNumber, false);
+				model.forceTokenization(lineNumber);
+				const lineTokens = model.getLineTokens(lineNumber);
 				for (let token = lineTokens.firstToken(); !!token; token = token.next()) {
 					const tokenStr = lineContent.substring(token.startOffset, token.endOffset);