More cleanup in tokens code

src/vs/editor/common/model/textModelTokens.ts

@@ -31,12 +31,12 @@ const enum Constants {
 }
 
 class ModelLineTokens {
-	public state: IState | null;
+	public beginState: IState | null;
 	public lineTokens: ArrayBuffer | null;
 	public valid: boolean;
 
-	constructor(state: IState | null) {
-		this.state = state;
+	constructor() {
+		this.beginState = null;
 		this.lineTokens = null;
 		this.valid = false;
 	}
@@ -167,18 +167,30 @@ class ModelLineTokens {
 			tokens[tokenIndex << 1] += textLength;
 		}
 	}
+
+	public setTokens(lineTokens: ArrayBuffer | null, valid: boolean): void {
+		this.lineTokens = lineTokens;
+		this.valid = valid;
+	}
+
+	public setBeginState(beginState: IState | null): void {
+		this.beginState = beginState;
+	}
+
+	public invalidate(): void {
+		this.valid = false;
+	}
 }
 
 export interface ITokensStore {
-	_invalidLineStartIndex: number;
 	readonly invalidLineStartIndex: number;
 
+	setGoodTokens(topLevelLanguageId: LanguageId, linesLength: number, lineIndex: number, lineTextLength: number, r: TokenizationResult2): void;
+	setFakeTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineTextLength: number, r: TokenizationResult2): void;
+
 	getTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineText: string): LineTokens;
-	invalidateLine(lineIndex: number): void;
-	getState(lineIndex: number): IState | null;
-	setTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineTextLength: number, tokens: Uint32Array): void;
-	setGoodTokens(topLevelLanguageId: LanguageId, linesLength: number, lineIndex: number, text: string, r: TokenizationResult2): void;
-	setState(lineIndex: number, state: IState): void;
+	getBeginState(lineIndex: number): IState | null;
+
 	applyEdits(range: Range, eolCount: number, firstLineLength: number): void;
 
 	_getAllStates(linesLength: number): (IState | null)[];
@@ -187,7 +199,7 @@ export interface ITokensStore {
 
 export class TokensStore implements ITokensStore {
 	private _tokens: ModelLineTokens[];
-	_invalidLineStartIndex: number;
+	private _invalidLineStartIndex: number;
 	private _lastState: IState | null;
 
 	constructor(initialState: IState | null) {
@@ -200,7 +212,7 @@ export class TokensStore implements ITokensStore {
 		this._lastState = null;
 
 		if (initialState) {
-			this._tokens[0] = new ModelLineTokens(initialState);
+			this._setBeginState(0, initialState);
 		}
 	}
 
@@ -210,7 +222,7 @@ export class TokensStore implements ITokensStore {
 
 	public getTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineText: string): LineTokens {
 		let rawLineTokens: ArrayBuffer | null = null;
-		if (lineIndex < this._tokens.length && this._tokens[lineIndex]) {
+		if (lineIndex < this._tokens.length) {
 			rawLineTokens = this._tokens[lineIndex].lineTokens;
 		}
 
@@ -224,43 +236,31 @@ export class TokensStore implements ITokensStore {
 		return new LineTokens(lineTokens, lineText);
 	}
 
-	public invalidateLine(lineIndex: number): void {
-		this._setIsValid(lineIndex, false);
-		if (lineIndex < this._invalidLineStartIndex) {
-			this._setIsValid(this._invalidLineStartIndex, false);
-			this._invalidLineStartIndex = lineIndex;
+	private _invalidateLine(lineIndex: number): void {
+		if (lineIndex < this._tokens.length) {
+			this._tokens[lineIndex].invalidate();
 		}
-	}
 
-	private _setIsValid(lineIndex: number, valid: boolean): void {
-		if (lineIndex < this._tokens.length && this._tokens[lineIndex]) {
-			this._tokens[lineIndex].valid = valid;
+		if (lineIndex < this._invalidLineStartIndex) {
+			this._invalidLineStartIndex = lineIndex;
 		}
 	}
 
 	private _isValid(lineIndex: number): boolean {
-		if (lineIndex < this._tokens.length && this._tokens[lineIndex]) {
+		if (lineIndex < this._tokens.length) {
 			return this._tokens[lineIndex].valid;
 		}
 		return false;
 	}
 
-	public getState(lineIndex: number): IState | null {
-		if (lineIndex < this._tokens.length && this._tokens[lineIndex]) {
-			return this._tokens[lineIndex].state;
+	public getBeginState(lineIndex: number): IState | null {
+		if (lineIndex < this._tokens.length) {
+			return this._tokens[lineIndex].beginState;
 		}
 		return null;
 	}
 
-	public setTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineTextLength: number, tokens: Uint32Array): void {
-		let target: ModelLineTokens;
-		if (lineIndex < this._tokens.length && this._tokens[lineIndex]) {
-			target = this._tokens[lineIndex];
-		} else {
-			target = new ModelLineTokens(null);
-			this._tokens[lineIndex] = target;
-		}
-
+	private static _massageTokens(topLevelLanguageId: LanguageId, lineTextLength: number, tokens: Uint32Array): ArrayBuffer {
 		if (lineTextLength === 0) {
 			let hasDifferentLanguageId = false;
 			if (tokens && tokens.length > 1) {
@@ -268,8 +268,7 @@ export class TokensStore implements ITokensStore {
 			}
 
 			if (!hasDifferentLanguageId) {
-				target.lineTokens = EMPTY_LINE_TOKENS;
-				return;
+				return EMPTY_LINE_TOKENS;
 			}
 		}
 
@@ -281,52 +280,62 @@ export class TokensStore implements ITokensStore {
 
 		LineTokens.convertToEndOffset(tokens, lineTextLength);
 
-		target.lineTokens = tokens.buffer;
-	}
-
-	public setGoodTokens(topLevelLanguageId: LanguageId, linesLength: number, lineIndex: number, text: string, r: TokenizationResult2): void {
-		const endStateIndex = lineIndex + 1;
-		this.setTokens(topLevelLanguageId, lineIndex, text.length, r.tokens);
-		this._setIsValid(lineIndex, true);
-
-		if (endStateIndex < linesLength) {
-			const previousEndState = this.getState(endStateIndex);
-			if (previousEndState !== null && r.endState.equals(previousEndState)) {
-				// The end state of this line remains the same
-				let nextInvalidLineIndex = lineIndex + 1;
-				while (nextInvalidLineIndex < linesLength) {
-					if (!this._isValid(nextInvalidLineIndex)) {
-						break;
-					}
-					if (nextInvalidLineIndex + 1 < linesLength) {
-						if (this.getState(nextInvalidLineIndex + 1) === null) {
-							break;
-						}
-					} else {
-						if (this._lastState === null) {
-							break;
-						}
-					}
-					nextInvalidLineIndex++;
-				}
-				this._invalidLineStartIndex = nextInvalidLineIndex;
-			} else {
-				this._invalidLineStartIndex = lineIndex + 1;
-				this.setState(endStateIndex, r.endState);
-			}
-		} else {
-			this._lastState = r.endState;
-			this._invalidLineStartIndex = linesLength;
+		return tokens.buffer;
+	}
+
+	private _getOrCreate(lineIndex: number): ModelLineTokens {
+		if (lineIndex < this._tokens.length) {
+			return this._tokens[lineIndex];
 		}
+		while (lineIndex > this._tokens.length) {
+			this._tokens[this._tokens.length] = new ModelLineTokens();
+		}
+		const result = new ModelLineTokens();
+		this._tokens[lineIndex] = result;
+		return result;
 	}
 
-	public setState(lineIndex: number, state: IState): void {
-		if (lineIndex < this._tokens.length && this._tokens[lineIndex]) {
-			this._tokens[lineIndex].state = state;
-		} else {
-			const tmp = new ModelLineTokens(state);
-			this._tokens[lineIndex] = tmp;
+	private _setTokens(lineIndex: number, tokens: ArrayBuffer | null, valid: boolean): void {
+		this._getOrCreate(lineIndex).setTokens(tokens, valid);
+	}
+
+	private _setBeginState(lineIndex: number, beginState: IState | null): void {
+		this._getOrCreate(lineIndex).setBeginState(beginState);
+	}
+
+	public setGoodTokens(topLevelLanguageId: LanguageId, linesLength: number, lineIndex: number, lineTextLength: number, r: TokenizationResult2): void {
+		const tokens = TokensStore._massageTokens(topLevelLanguageId, lineTextLength, r.tokens);
+		this._setTokens(lineIndex, tokens, true);
+		this._invalidLineStartIndex = lineIndex + 1;
+
+		// Check if this was the last line
+		if (lineIndex === linesLength - 1) {
+			this._lastState = r.endState;
+			return;
 		}
+
+		// Check if the end state has changed
+		const previousEndState = this.getBeginState(lineIndex + 1);
+		if (previousEndState === null || !r.endState.equals(previousEndState)) {
+			this._setBeginState(lineIndex + 1, r.endState);
+			this._invalidateLine(lineIndex + 1);
+			return;
+		}
+
+		// Perhaps we can skip tokenizing some lines...
+		let i = lineIndex + 1;
+		while (i < linesLength) {
+			if (!this._isValid(i)) {
+				break;
+			}
+			i++;
+		}
+		this._invalidLineStartIndex = i;
+	}
+
+	setFakeTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineTextLength: number, r: TokenizationResult2): void {
+		const tokens = TokensStore._massageTokens(topLevelLanguageId, lineTextLength, r.tokens);
+		this._setTokens(lineIndex, tokens, false);
 	}
 
 	//#region Editing
@@ -338,14 +347,14 @@ export class TokensStore implements ITokensStore {
 			const editingLinesCnt = Math.min(deletingLinesCnt, insertingLinesCnt);
 
 			for (let j = editingLinesCnt; j >= 0; j--) {
-				this.invalidateLine(range.startLineNumber + j - 1);
+				this._invalidateLine(range.startLineNumber + j - 1);
 			}
 
 			this._acceptDeleteRange(range);
 			this._acceptInsertText(new Position(range.startLineNumber, range.startColumn), eolCount, firstLineLength);
 		} catch (err) {
 			// emergency recovery => reset tokens
-			this._reset(this.getState(0));
+			this._reset(this.getBeginState(0));
 		}
 	}
 
@@ -407,8 +416,8 @@ export class TokensStore implements ITokensStore {
 		line.insert(position.column - 1, firstLineLength);
 
 		let insert: ModelLineTokens[] = new Array<ModelLineTokens>(eolCount);
-		for (let i = eolCount - 1; i >= 0; i--) {
-			insert[i] = new ModelLineTokens(null);
+		for (let i = 0; i < eolCount; i++) {
+			insert[i] = new ModelLineTokens();
 		}
 		this._tokens = arrays.arrayInsert(this._tokens, position.lineNumber, insert);
 	}
@@ -418,7 +427,7 @@ export class TokensStore implements ITokensStore {
 	_getAllStates(linesLength: number): (IState | null)[] {
 		const r: (IState | null)[] = [];
 		for (let i = 0; i < linesLength; i++) {
-			r[i] = this.getState(i);
+			r[i] = this.getBeginState(i);
 		}
 		r[linesLength] = this._lastState;
 		return r;
@@ -457,6 +466,10 @@ export class ModelLinesTokens implements IModelLinesTokens {
 	}
 
 	public isCheapToTokenize(store: ITokensStore, buffer: ITextBuffer, lineNumber: number): boolean {
+		if (!this.tokenizationSupport) {
+			return true;
+		}
+
 		const firstInvalidLineNumber = store.invalidLineStartIndex + 1;
 		if (lineNumber > firstInvalidLineNumber) {
 			return false;
@@ -474,6 +487,10 @@ export class ModelLinesTokens implements IModelLinesTokens {
 	}
 
 	public hasLinesToTokenize(store: ITokensStore, buffer: ITextBuffer): boolean {
+		if (!this.tokenizationSupport) {
+			return false;
+		}
+
 		return (store.invalidLineStartIndex < buffer.getLineCount());
 	}
 
@@ -490,7 +507,6 @@ export class ModelLinesTokens implements IModelLinesTokens {
 
 	public updateTokensUntilLine(store: ITokensStore, buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder, lineNumber: number): void {
 		if (!this.tokenizationSupport) {
-			store._invalidLineStartIndex = buffer.getLineCount();
 			return;
 		}
 
@@ -500,10 +516,10 @@ export class ModelLinesTokens implements IModelLinesTokens {
 		// Validate all states up to and including endLineIndex
 		for (let lineIndex = store.invalidLineStartIndex; lineIndex <= endLineIndex; lineIndex++) {
 			const text = buffer.getLineContent(lineIndex + 1);
-			const lineStartState = store.getState(lineIndex);
+			const lineStartState = store.getBeginState(lineIndex);
 
 			const r = safeTokenize(this._languageIdentifier, this.tokenizationSupport, text, lineStartState!);
-			store.setGoodTokens(this._languageIdentifier.id, linesLength, lineIndex, text, r);
+			store.setGoodTokens(this._languageIdentifier.id, linesLength, lineIndex, text.length, r);
 			eventBuilder.registerChangedTokens(lineIndex + 1);
 			lineIndex = store.invalidLineStartIndex - 1; // -1 because the outer loop increments it
 		}
@@ -537,7 +553,7 @@ export class ModelLinesTokens implements IModelLinesTokens {
 			}
 
 			if (newNonWhitespaceIndex < nonWhitespaceColumn) {
-				initialState = store.getState(i - 1);
+				initialState = store.getBeginState(i - 1);
 				if (initialState) {
 					break;
 				}
@@ -553,37 +569,15 @@ export class ModelLinesTokens implements IModelLinesTokens {
 		let state = initialState;
 		for (let i = fakeLines.length - 1; i >= 0; i--) {
 			let r = safeTokenize(this._languageIdentifier, this.tokenizationSupport, fakeLines[i], state);
-			if (r) {
-				state = r.endState;
-			} else {
-				state = initialState;
-			}
+			state = r.endState;
 		}
 
-		this._fakeTokenizeLines(store, buffer, eventBuilder, state, startLineNumber, endLineNumber);
-	}
-
-	private _fakeTokenizeLines(store: ITokensStore, buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder, initialState: IState, startLineNumber: number, endLineNumber: number): void {
-		if (!this.tokenizationSupport) {
-			return;
-		}
-
-		let state = initialState;
-		for (let i = startLineNumber; i <= endLineNumber; i++) {
-			let text = buffer.getLineContent(i);
+		for (let lineNumber = startLineNumber; lineNumber <= endLineNumber; lineNumber++) {
+			let text = buffer.getLineContent(lineNumber);
 			let r = safeTokenize(this._languageIdentifier, this.tokenizationSupport, text, state);
-			if (r) {
-				store.setTokens(this._languageIdentifier.id, i - 1, text.length, r.tokens);
-
-				// We cannot trust these states/tokens to be valid!
-				// (see https://github.com/Microsoft/vscode/issues/67607)
-				store.invalidateLine(i - 1);
-				store.setState(i - 1, state);
-				state = r.endState;
-				eventBuilder.registerChangedTokens(i);
-			} else {
-				state = initialState;
-			}
+			store.setFakeTokens(this._languageIdentifier.id, lineNumber - 1, text.length, r);
+			state = r.endState;
+			eventBuilder.registerChangedTokens(lineNumber);
 		}
 	}
 

src/vs/editor/test/common/model/model.line.test.ts

@@ -9,6 +9,8 @@ import { Range } from 'vs/editor/common/core/range';
 import { TextModel } from 'vs/editor/common/model/textModel';
 import { LanguageIdentifier, MetadataConsts } from 'vs/editor/common/modes';
 import { ViewLineToken, ViewLineTokenFactory } from 'vs/editor/test/common/core/viewLineToken';
+import { TokenizationResult2 } from 'vs/editor/common/core/token';
+import { NULL_STATE } from 'vs/editor/common/modes/nullMode';
 
 interface ILineEdit {
 	startColumn: number;
@@ -92,6 +94,10 @@ class TestToken {
 	}
 }
 
+function toTokenizationResult2(tokens: Uint32Array): TokenizationResult2 {
+	return new TokenizationResult2(tokens, NULL_STATE);
+}
+
 suite('ModelLinesTokens', () => {
 
 	interface IBufferLineState {
@@ -110,7 +116,7 @@ suite('ModelLinesTokens', () => {
 		for (let lineIndex = 0; lineIndex < initial.length; lineIndex++) {
 			const lineTokens = initial[lineIndex].tokens;
 			const lineTextLength = model.getLineMaxColumn(lineIndex + 1) - 1;
-			model._tokens.setTokens(0, lineIndex, lineTextLength, TestToken.toTokens(lineTokens));
+			model._tokens.setFakeTokens(0, lineIndex, lineTextLength, toTokenizationResult2(TestToken.toTokens(lineTokens)));
 		}
 
 		model.applyEdits(edits.map((ed) => ({
@@ -441,14 +447,14 @@ suite('ModelLinesTokens', () => {
 
 	test('insertion on empty line', () => {
 		const model = new TextModel('some text', TextModel.DEFAULT_CREATION_OPTIONS, new LanguageIdentifier('test', 0));
-		model._tokens.setTokens(0, 0, model.getLineMaxColumn(1) - 1, TestToken.toTokens([new TestToken(0, 1)]));
+		model._tokens.setFakeTokens(0, 0, model.getLineMaxColumn(1) - 1, toTokenizationResult2(TestToken.toTokens([new TestToken(0, 1)])));
 
 		model.applyEdits([{
 			range: new Range(1, 1, 1, 10),
 			text: ''
 		}]);
 
-		model._tokens.setTokens(0, 0, model.getLineMaxColumn(1) - 1, new Uint32Array(0));
+		model._tokens.setFakeTokens(0, 0, model.getLineMaxColumn(1) - 1, toTokenizationResult2(new Uint32Array(0)));
 
 		model.applyEdits([{
 			range: new Range(1, 1, 1, 1),