Invalidate tokens correctly

src/vs/editor/common/model/editableTextModel.ts

@@ -202,6 +202,10 @@ export class EditableTextModel extends TextModelWithDecorations implements edito
 			);
 		}
 
+		if (this._tokens.hasLinesToTokenize(this._buffer)) {
+			this._beginBackgroundTokenization();
+		}
+
 		return result.reverseEdits;
 	}
 

src/vs/editor/common/model/modelLine.ts

@@ -257,12 +257,10 @@ export class ModelLinesTokens {
 		return (this._invalidLineStartIndex < buffer.getLineCount());
 	}
 
-	public invalidateLine(buffer: TextBuffer, lineIndex: number): void {
+	public invalidateLine(lineIndex: number): void {
 		this._setIsInvalid(lineIndex, true);
 		if (lineIndex < this._invalidLineStartIndex) {
-			if (this._invalidLineStartIndex < buffer.getLineCount()) {
-				this._setIsInvalid(this._invalidLineStartIndex, true);
-			}
+			this._setIsInvalid(this._invalidLineStartIndex, true);
 			this._invalidLineStartIndex = lineIndex;
 		}
 	}
@@ -323,7 +321,32 @@ export class ModelLinesTokens {
 
 	//#region Editing
 
+	// TODO: simplify
 	public applyEdits(range: Range, lines: string[]): void {
+
+		const deletingLinesCnt = range.endLineNumber - range.startLineNumber;
+		const insertingLinesCnt = (lines ? lines.length - 1 : 0);
+		const editingLinesCnt = Math.min(deletingLinesCnt, insertingLinesCnt);
+
+		// Iterating descending to overlap with previous op
+		// in case there are common lines being edited in both
+		for (let j = editingLinesCnt; j >= 0; j--) {
+			const editLineNumber = range.startLineNumber + j;
+			this.invalidateLine(editLineNumber - 1);
+		}
+
+		if (editingLinesCnt < deletingLinesCnt) {
+			// Must delete some lines
+			const spliceStartLineNumber = range.startLineNumber + editingLinesCnt;
+			this.invalidateLine(spliceStartLineNumber - 1);
+		}
+
+		if (editingLinesCnt < insertingLinesCnt) {
+			// Must insert some lines
+			const spliceLineNumber = range.startLineNumber + editingLinesCnt;
+			this.invalidateLine(spliceLineNumber - 1);
+		}
+
 		this._acceptDeleteRange(range);
 		this._acceptInsertText(new Position(range.startLineNumber, range.startColumn), lines);
 	}

src/vs/editor/common/model/textBuffer.ts

@@ -464,6 +464,7 @@ export class TextBuffer {
 		};
 	}
 
+	// TODO: simplify
 	private _doApplyEdits(operations: IValidatedEditOperation[]): [ModelRawChange[], IInternalModelContentChange[]] {
 
 		// Sort operations descending
@@ -499,7 +500,6 @@ export class TextBuffer {
 					continue;
 				}
 
-				// this._invalidateLine(currentLineNumber - 1); //TODO@TextBuffer
 				this._lines[currentLineNumber - 1] = applyLineEdits(
 					this._lines[currentLineNumber - 1],
 					lineEditsQueue.slice(currentLineNumberStart, i)
@@ -513,7 +513,6 @@ export class TextBuffer {
 				currentLineNumberStart = i;
 			}
 
-			// this._invalidateLine(currentLineNumber - 1); //TODO@TextBuffer
 			this._lines[currentLineNumber - 1] = applyLineEdits(
 				this._lines[currentLineNumber - 1],
 				lineEditsQueue.slice(currentLineNumberStart, lineEditsQueue.length)
@@ -573,7 +572,6 @@ export class TextBuffer {
 				const [t1, t2] = splitLine(this._lines[endLineNumber - 1], endColumn);
 				this._lines[endLineNumber - 1] = t1;
 				const endLineRemains = t2;
-				// this._invalidateLine(spliceStartLineNumber - 1); //TODO@TextBuffer
 
 				const spliceCnt = endLineNumber - spliceStartLineNumber;
 
@@ -613,7 +611,6 @@ export class TextBuffer {
 				rawContentChanges.push(
 					new ModelRawLineChanged(spliceLineNumber, this._lines[spliceLineNumber - 1])
 				);
-				// this._invalidateLine(spliceLineNumber - 1); //TODO@TextBuffer
 
 				// Lines in the middle
 				let newLines: string[] = [];

src/vs/editor/common/model/textModelWithTokens.ts

@@ -193,14 +193,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 		return lineTokens.getLanguageId(lineTokens.findTokenIndexAtOffset(column - 1));
 	}
 
-	protected _invalidateLine(lineIndex: number): void {
-		this._tokens.invalidateLine(this._buffer, lineIndex);
-		if (this._tokens.hasLinesToTokenize(this._buffer)) {
-			this._beginBackgroundTokenization();
-		}
-	}
-
-	private _beginBackgroundTokenization(): void {
+	protected _beginBackgroundTokenization(): void {
 		if (this._shouldAutoTokenize() && this._revalidateTokensTimeout === -1) {
 			this._revalidateTokensTimeout = setTimeout(() => {
 				this._revalidateTokensTimeout = -1;

src/vs/editor/test/common/model/model.modes.test.ts

@@ -201,7 +201,7 @@ suite('Editor Model - Model Modes 2', () => {
 		for (i = 0; i < len; i++) {
 			stateEqual(model._tokens._getState(i), states[i]);
 		}
-		stateEqual((<any>model)._lastState, states[len]);
+		stateEqual((<any>model)._tokens._lastState, states[len]);
 	}
 
 	let thisModel: Model = null;