tokenize viewport before we render.

src/vs/editor/browser/widget/codeEditorWidget.ts

@@ -25,7 +25,7 @@ import { Event, Emitter } from 'vs/base/common/event';
 import { IKeyboardEvent } from 'vs/base/browser/keyboardEvent';
 import { InternalEditorAction } from 'vs/editor/common/editorAction';
 import { IEditorOptions } from 'vs/editor/common/config/editorOptions';
-import { IPosition } from 'vs/editor/common/core/position';
+import { IPosition, Position } from 'vs/editor/common/core/position';
 import { CoreEditorCommand } from 'vs/editor/browser/controller/coreCommands';
 import { IThemeService, registerThemingParticipant } from 'vs/platform/theme/common/themeService';
 import { editorErrorForeground, editorErrorBorder, editorWarningForeground, editorWarningBorder, editorInfoBorder, editorInfoForeground, editorHintForeground, editorHintBorder } from 'vs/editor/common/view/editorColorRegistry';
@@ -455,7 +455,12 @@ export abstract class CodeEditorWidget extends CommonCodeEditor implements edito
 			return;
 		}
 		if (s && s.cursorState && s.viewState) {
-			this._view.restoreState(this.viewModel.viewLayout.reduceRestoreState(s.viewState));
+			const reducedState = this.viewModel.viewLayout.reduceRestoreState(s.viewState);
+			const linesViewportData = this.viewModel.viewLayout.getLinesViewportDataAtScrollTop(reducedState.scrollTop);
+			const startViewPosition = this.viewModel.coordinatesConverter.convertViewPositionToModelPosition(new Position(linesViewportData.startLineNumber, 1));
+			const endViewPosition = this.viewModel.coordinatesConverter.convertViewPositionToModelPosition(new Position(linesViewportData.endLineNumber, 1));
+			this.model.tokenizeViewport(startViewPosition.lineNumber, endViewPosition.lineNumber);
+			this._view.restoreState(reducedState);
 		}
 	}
 

src/vs/editor/common/model.ts

@@ -650,6 +650,11 @@ export interface ITextModel {
 	 */
 	isDisposed(): boolean;
 
+	/**
+	 * @internal
+	 */
+	tokenizeViewport(startLineNumber: number, endLineNumber: number): void;
+
 	/**
 	 * Only basic mode supports allowed on this model because it is simply too large.
 	 * (tokenization is allowed and other basic supports)

src/vs/editor/common/model/textModel.ts

@@ -1625,6 +1625,67 @@ export class TextModel extends Disposable implements model.ITextModel {
 
 	//#region Tokenization
 
+	public tokenizeViewport(startLineNumber: number, endLineNumber: number): void {
+		if (!this._tokens.tokenizationSupport) {
+			return;
+		}
+
+		const eventBuilder = new ModelTokensChangedEventBuilder();
+		const viewPortLimit = 120;
+		const context = Math.floor(Math.max(viewPortLimit - (endLineNumber - startLineNumber), 0) / 2);
+		startLineNumber = Math.max(1, startLineNumber - context);
+		let nonWhitespaceColumn = this.getLineFirstNonWhitespaceColumn(startLineNumber);
+
+		let fakeLines = [];
+		let i = startLineNumber - 1;
+		if (nonWhitespaceColumn > 0) {
+			while (nonWhitespaceColumn > 0 && i >= 1) {
+				let newNonWhitespaceIndex = this.getLineFirstNonWhitespaceColumn(i);
+
+				if (newNonWhitespaceIndex === 0) {
+					i--;
+					continue;
+				}
+
+				if (newNonWhitespaceIndex < nonWhitespaceColumn) {
+					fakeLines.push(this.getLineContent(i));
+					nonWhitespaceColumn = newNonWhitespaceIndex;
+				}
+
+				i--;
+			}
+		}
+
+		const initialState = this._tokens.tokenizationSupport.getInitialState();
+		let state = initialState.clone();
+		for (let i = fakeLines.length - 1; i >= 0; i--) {
+			let r = this._tokens._tokenizeOneLine2(this._buffer, fakeLines[i], state, eventBuilder);
+			if (r) {
+				state = r.endState.clone();
+			} else {
+				state = initialState.clone();
+			}
+		}
+
+		endLineNumber = Math.min(this.getLineCount(), endLineNumber + context);
+		for (let i = startLineNumber; i <= endLineNumber; i++) {
+			let text = this.getLineContent(i);
+			let r = this._tokens._tokenizeOneLine2(this._buffer, text, state, eventBuilder);
+			if (r) {
+				state = r.endState.clone();
+				this._tokens._setTokens(this._tokens.languageIdentifier.id, i - 1, text.length, r.tokens);
+				eventBuilder.registerChangedTokens(i);
+			} else {
+				state = initialState.clone();
+			}
+		}
+
+		const e = eventBuilder.build();
+		if (e) {
+			this._onDidChangeTokens.fire(e);
+		}
+	}
+
 	public forceTokenization(lineNumber: number): void {
 		if (lineNumber < 1 || lineNumber > this.getLineCount()) {
 			throw new Error('Illegal value for lineNumber');

src/vs/editor/common/model/textModelTokens.ts

@@ -198,7 +198,7 @@ export class ModelLinesTokens {
 
 	public getTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineText: string): LineTokens {
 		let rawLineTokens: ArrayBuffer = null;
-		if (lineIndex < this._tokens.length) {
+		if (lineIndex < this._tokens.length && this._tokens[lineIndex]) {
 			rawLineTokens = this._tokens[lineIndex]._lineTokens;
 		}
 
@@ -230,20 +230,20 @@ export class ModelLinesTokens {
 	}
 
 	private _setIsInvalid(lineIndex: number, invalid: boolean): void {
-		if (lineIndex < this._tokens.length) {
+		if (lineIndex < this._tokens.length && this._tokens[lineIndex]) {
 			this._tokens[lineIndex]._invalid = invalid;
 		}
 	}
 
 	_isInvalid(lineIndex: number): boolean {
-		if (lineIndex < this._tokens.length) {
+		if (lineIndex < this._tokens.length && this._tokens[lineIndex]) {
 			return this._tokens[lineIndex]._invalid;
 		}
 		return true;
 	}
 
 	_getState(lineIndex: number): IState {
-		if (lineIndex < this._tokens.length) {
+		if (lineIndex < this._tokens.length && this._tokens[lineIndex]) {
 			return this._tokens[lineIndex]._state;
 		}
 		return null;
@@ -251,7 +251,7 @@ export class ModelLinesTokens {
 
 	_setTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineTextLength: number, tokens: Uint32Array): void {
 		let target: ModelLineTokens;
-		if (lineIndex < this._tokens.length) {
+		if (lineIndex < this._tokens.length && this._tokens[lineIndex]) {
 			target = this._tokens[lineIndex];
 		} else {
 			target = new ModelLineTokens(null);
@@ -275,7 +275,7 @@ export class ModelLinesTokens {
 	}
 
 	private _setState(lineIndex: number, state: IState): void {
-		if (lineIndex < this._tokens.length) {
+		if (lineIndex < this._tokens.length && this._tokens[lineIndex]) {
 			this._tokens[lineIndex]._state = state;
 		} else {
 			const tmp = new ModelLineTokens(state);
@@ -376,6 +376,25 @@ export class ModelLinesTokens {
 		return lineNumber;
 	}
 
+	public _tokenizeOneLine2(buffer: ITextBuffer, text: string, state: IState, eventBuilder: ModelTokensChangedEventBuilder): TokenizationResult2 {
+		if (!this.hasLinesToTokenize(buffer)) {
+			return null;
+		}
+
+		let r: TokenizationResult2 = null;
+
+		try {
+			r = this.tokenizationSupport.tokenize2(text, state, 0);
+		} catch (e) {
+			onUnexpectedError(e);
+		}
+
+		if (!r) {
+			r = nullTokenize2(this.languageIdentifier.id, text, state, 0);
+		}
+		return r;
+	}
+
 	public _updateTokensUntilLine(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder, lineNumber: number): void {
 		if (!this.tokenizationSupport) {
 			this._invalidLineStartIndex = buffer.getLineCount();