diff --git a/src/vs/editor/common/model/textModel.ts b/src/vs/editor/common/model/textModel.ts index 0ed04c270de71e184d07d9284dc1f911fc7722d7..6edd0f4ecfb6672933e5c3fcc3ae45826d9ed23b 100644 --- a/src/vs/editor/common/model/textModel.ts +++ b/src/vs/editor/common/model/textModel.ts @@ -32,7 +32,7 @@ import { BracketsUtils, RichEditBracket, RichEditBrackets } from 'vs/editor/comm import { ITheme, ThemeColor } from 'vs/platform/theme/common/themeService'; import { withUndefinedAsNull } from 'vs/base/common/types'; import { VSBufferReadableStream, VSBuffer } from 'vs/base/common/buffer'; -import { TokensStore } from 'vs/editor/common/model/tokensStore'; +import { TokensStore, MultilineTokens } from 'vs/editor/common/model/tokensStore'; function createTextBufferBuilder() { return new PieceTreeTextBufferBuilder(); @@ -424,6 +424,9 @@ export class TextModel extends Disposable implements model.ITextModel { this._buffer = textBuffer; this._increaseVersionId(); + // Flush all tokens + this._tokens.flush(); + // Destroy all my decorations this._decorations = Object.create(null); this._decorationsTree = new DecorationsTrees(); @@ -1708,12 +1711,48 @@ export class TextModel extends Disposable implements model.ITextModel { this._tokens.setTokens(this._languageIdentifier.id, lineNumber - 1, this._buffer.getLineLength(lineNumber), tokens); } + public setTokens(tokens: MultilineTokens[]): void { + if (tokens.length === 0) { + return; + } + + let ranges: { fromLineNumber: number; toLineNumber: number; }[] = []; + + for (let i = 0, len = tokens.length; i < len; i++) { + const element = tokens[i]; + ranges.push({ fromLineNumber: element.startLineNumber, toLineNumber: element.startLineNumber + element.tokens.length - 1 }); + for (let j = 0, lenJ = element.tokens.length; j < lenJ; j++) { + this.setLineTokens(element.startLineNumber + j, element.tokens[j]); + } + } + + this._emitModelTokensChangedEvent({ + tokenizationSupportChanged: false, + ranges: ranges + }); + } + public tokenizeViewport(startLineNumber: number, endLineNumber: number): void { + startLineNumber = Math.max(1, startLineNumber); + endLineNumber = Math.min(this._buffer.getLineCount(), endLineNumber); this._tokenization.tokenizeViewport(startLineNumber, endLineNumber); } public clearTokens(): void { this._tokens.flush(); + this._emitModelTokensChangedEvent({ + tokenizationSupportChanged: true, + ranges: [{ + fromLineNumber: 1, + toLineNumber: this._buffer.getLineCount() + }] + }); + } + + private _emitModelTokensChangedEvent(e: IModelTokensChangedEvent): void { + if (!this._isDisposing) { + this._onDidChangeTokens.fire(e); + } } public resetTokenization(): void { @@ -1782,12 +1821,6 @@ export class TextModel extends Disposable implements model.ITextModel { return lineTokens.getLanguageId(lineTokens.findTokenIndexAtOffset(position.column - 1)); } - emitModelTokensChangedEvent(e: IModelTokensChangedEvent): void { - if (!this._isDisposing) { - this._onDidChangeTokens.fire(e); - } - } - // Having tokens allows implementing additional helper methods public getWordAtPosition(_position: IPosition): model.IWordAtPosition | null { diff --git a/src/vs/editor/common/model/textModelTokens.ts b/src/vs/editor/common/model/textModelTokens.ts index 75fd1c6aae3eb1165eae37ab067a987ca24f39f6..b959ba7a1b6d8cfb6572f9868c3d28ab54ff5a30 100644 --- a/src/vs/editor/common/model/textModelTokens.ts +++ b/src/vs/editor/common/model/textModelTokens.ts @@ -9,13 +9,14 @@ import { LineTokens } from 'vs/editor/common/core/lineTokens'; import { Position } from 'vs/editor/common/core/position'; import { IRange } from 'vs/editor/common/core/range'; import { TokenizationResult2 } from 'vs/editor/common/core/token'; -import { IModelTokensChangedEvent, RawContentChangedType } from 'vs/editor/common/model/textModelEvents'; +import { RawContentChangedType } from 'vs/editor/common/model/textModelEvents'; import { IState, ITokenizationSupport, LanguageIdentifier, TokenizationRegistry } from 'vs/editor/common/modes'; import { nullTokenize2 } from 'vs/editor/common/modes/nullMode'; import { TextModel } from 'vs/editor/common/model/textModel'; import { Disposable } from 'vs/base/common/lifecycle'; import { StopWatch } from 'vs/base/common/stopwatch'; import { CharCode } from 'vs/base/common/charCode'; +import { MultilineTokensBuilder } from 'vs/editor/common/model/tokensStore'; export function countEOL(text: string): [number, number] { let eolCount = 0; @@ -146,7 +147,7 @@ export class TokenizationStateStore { this._beginState[lineIndex] = beginState; } - public setGoodTokens(linesLength: number, lineIndex: number, endState: IState): void { + public setEndState(linesLength: number, lineIndex: number, endState: IState): void { this._setValid(lineIndex, true); this._invalidLineStartIndex = lineIndex + 1; @@ -238,19 +239,11 @@ export class TextModelTokenization extends Disposable { this._resetTokenizationState(); this._textModel.clearTokens(); - this._textModel.emitModelTokensChangedEvent({ - tokenizationSupportChanged: true, - ranges: [{ - fromLineNumber: 1, - toLineNumber: this._textModel.getLineCount() - }] - }); })); this._register(this._textModel.onDidChangeRawContentFast((e) => { if (e.containsEvent(RawContentChangedType.Flush)) { this._resetTokenizationState(); - this._textModel.clearTokens(); return; } })); @@ -272,14 +265,6 @@ export class TextModelTokenization extends Disposable { this._register(this._textModel.onDidChangeLanguage(() => { this._resetTokenizationState(); this._textModel.clearTokens(); - - this._textModel.emitModelTokensChangedEvent({ - tokenizationSupportChanged: true, - ranges: [{ - fromLineNumber: 1, - toLineNumber: this._textModel.getLineCount() - }] - }); })); this._resetTokenizationState(); @@ -316,7 +301,7 @@ export class TextModelTokenization extends Disposable { private _revalidateTokensNow(toLineNumber: number = this._textModel.getLineCount()): void { const MAX_ALLOWED_TIME = 20; - const eventBuilder = new ModelTokensChangedEventBuilder(); + const builder = new MultilineTokensBuilder(); const sw = StopWatch.create(false); while (this._hasLinesToTokenize()) { @@ -325,7 +310,7 @@ export class TextModelTokenization extends Disposable { break; } - const tokenizedLineNumber = this._tokenizeOneInvalidLine(eventBuilder); + const tokenizedLineNumber = this._tokenizeOneInvalidLine(builder); if (tokenizedLineNumber >= toLineNumber) { break; @@ -333,47 +318,24 @@ export class TextModelTokenization extends Disposable { } this._beginBackgroundTokenization(); - - const e = eventBuilder.build(); - if (e) { - this._textModel.emitModelTokensChangedEvent(e); - } + this._textModel.setTokens(builder.tokens); } public tokenizeViewport(startLineNumber: number, endLineNumber: number): void { - startLineNumber = Math.max(1, startLineNumber); - endLineNumber = Math.min(this._textModel.getLineCount(), endLineNumber); - - const eventBuilder = new ModelTokensChangedEventBuilder(); - this._tokenizeViewport(eventBuilder, startLineNumber, endLineNumber); - - const e = eventBuilder.build(); - if (e) { - this._textModel.emitModelTokensChangedEvent(e); - } + const builder = new MultilineTokensBuilder(); + this._tokenizeViewport(builder, startLineNumber, endLineNumber); + this._textModel.setTokens(builder.tokens); } public reset(): void { this._resetTokenizationState(); this._textModel.clearTokens(); - this._textModel.emitModelTokensChangedEvent({ - tokenizationSupportChanged: false, - ranges: [{ - fromLineNumber: 1, - toLineNumber: this._textModel.getLineCount() - }] - }); } public forceTokenization(lineNumber: number): void { - const eventBuilder = new ModelTokensChangedEventBuilder(); - - this._updateTokensUntilLine(eventBuilder, lineNumber); - - const e = eventBuilder.build(); - if (e) { - this._textModel.emitModelTokensChangedEvent(e); - } + const builder = new MultilineTokensBuilder(); + this._updateTokensUntilLine(builder, lineNumber); + this._textModel.setTokens(builder.tokens); } public isCheapToTokenize(lineNumber: number): boolean { @@ -404,16 +366,16 @@ export class TextModelTokenization extends Disposable { return (this._tokenizationStateStore.invalidLineStartIndex < this._textModel.getLineCount()); } - private _tokenizeOneInvalidLine(eventBuilder: ModelTokensChangedEventBuilder): number { + private _tokenizeOneInvalidLine(builder: MultilineTokensBuilder): number { if (!this._hasLinesToTokenize()) { return this._textModel.getLineCount() + 1; } const lineNumber = this._tokenizationStateStore.invalidLineStartIndex + 1; - this._updateTokensUntilLine(eventBuilder, lineNumber); + this._updateTokensUntilLine(builder, lineNumber); return lineNumber; } - private _updateTokensUntilLine(eventBuilder: ModelTokensChangedEventBuilder, lineNumber: number): void { + private _updateTokensUntilLine(builder: MultilineTokensBuilder, lineNumber: number): void { if (!this._tokenizationSupport) { return; } @@ -427,14 +389,13 @@ export class TextModelTokenization extends Disposable { const lineStartState = this._tokenizationStateStore.getBeginState(lineIndex); const r = safeTokenize(languageIdentifier, this._tokenizationSupport, text, lineStartState!); - this._textModel.setLineTokens(lineIndex + 1, r.tokens); - this._tokenizationStateStore.setGoodTokens(linesLength, lineIndex, r.endState); - eventBuilder.registerChangedTokens(lineIndex + 1); + builder.add(lineIndex + 1, r.tokens); + this._tokenizationStateStore.setEndState(linesLength, lineIndex, r.endState); lineIndex = this._tokenizationStateStore.invalidLineStartIndex - 1; // -1 because the outer loop increments it } } - private _tokenizeViewport(eventBuilder: ModelTokensChangedEventBuilder, startLineNumber: number, endLineNumber: number): void { + private _tokenizeViewport(builder: MultilineTokensBuilder, startLineNumber: number, endLineNumber: number): void { if (!this._tokenizationSupport) { // nothing to do return; @@ -447,7 +408,7 @@ export class TextModelTokenization extends Disposable { if (startLineNumber <= this._tokenizationStateStore.invalidLineStartIndex) { // tokenization has reached the viewport start... - this._updateTokensUntilLine(eventBuilder, endLineNumber); + this._updateTokensUntilLine(builder, endLineNumber); return; } @@ -485,10 +446,9 @@ export class TextModelTokenization extends Disposable { for (let lineNumber = startLineNumber; lineNumber <= endLineNumber; lineNumber++) { let text = this._textModel.getLineContent(lineNumber); let r = safeTokenize(languageIdentifier, this._tokenizationSupport, text, state); - this._textModel.setLineTokens(lineNumber, r.tokens); + builder.add(lineNumber, r.tokens); this._tokenizationStateStore.setFakeTokens(lineNumber - 1); state = r.endState; - eventBuilder.registerChangedTokens(lineNumber); } } } @@ -530,39 +490,3 @@ function safeTokenize(languageIdentifier: LanguageIdentifier, tokenizationSuppor LineTokens.convertToEndOffset(r.tokens, text.length); return r; } - -export class ModelTokensChangedEventBuilder { - - private readonly _ranges: { fromLineNumber: number; toLineNumber: number; }[]; - - constructor() { - this._ranges = []; - } - - public registerChangedTokens(lineNumber: number): void { - const ranges = this._ranges; - const rangesLength = ranges.length; - const previousRange = rangesLength > 0 ? ranges[rangesLength - 1] : null; - - if (previousRange && previousRange.toLineNumber === lineNumber - 1) { - // extend previous range - previousRange.toLineNumber++; - } else { - // insert new range - ranges[rangesLength] = { - fromLineNumber: lineNumber, - toLineNumber: lineNumber - }; - } - } - - public build(): IModelTokensChangedEvent | null { - if (this._ranges.length === 0) { - return null; - } - return { - tokenizationSupportChanged: false, - ranges: this._ranges - }; - } -} diff --git a/src/vs/editor/common/model/tokensStore.ts b/src/vs/editor/common/model/tokensStore.ts index daad45e1f8393131effabe65bf497a21738a5c3b..8377ce07c3a060d5e44b809e25b0102098355614 100644 --- a/src/vs/editor/common/model/tokensStore.ts +++ b/src/vs/editor/common/model/tokensStore.ts @@ -21,6 +21,39 @@ function getDefaultMetadata(topLevelLanguageId: LanguageId): number { const EMPTY_LINE_TOKENS = (new Uint32Array(0)).buffer; +export class MultilineTokensBuilder { + + public readonly tokens: MultilineTokens[]; + + constructor() { + this.tokens = []; + } + + public add(lineNumber: number, lineTokens: Uint32Array): void { + if (this.tokens.length > 0) { + const last = this.tokens[this.tokens.length - 1]; + const lastLineNumber = last.startLineNumber + last.tokens.length - 1; + if (lastLineNumber + 1 === lineNumber) { + // append + last.tokens.push(lineTokens); + return; + } + } + this.tokens.push(new MultilineTokens(lineNumber, lineTokens)); + } +} + +export class MultilineTokens { + + public readonly startLineNumber: number; + public readonly tokens: Uint32Array[]; + + constructor(lineNumber: number, tokens: Uint32Array) { + this.startLineNumber = lineNumber; + this.tokens = [tokens]; + } +} + export class TokensStore { private _lineTokens: (ArrayBuffer | null)[]; private _len: number; @@ -65,7 +98,7 @@ export class TokensStore { if (!tokens || tokens.length === 0) { tokens = new Uint32Array(2); - tokens[0] = 0; + tokens[0] = lineTextLength; tokens[1] = getDefaultMetadata(topLevelLanguageId); } diff --git a/src/vs/workbench/services/textMate/electron-browser/textMateService.ts b/src/vs/workbench/services/textMate/electron-browser/textMateService.ts index d86016f64eea33371d712facb93f16fdb24b2527..9e301b37d855438732aa26012b9a57fd1a6a4068 100644 --- a/src/vs/workbench/services/textMate/electron-browser/textMateService.ts +++ b/src/vs/workbench/services/textMate/electron-browser/textMateService.ts @@ -75,6 +75,7 @@ class ModelWorkerTextMateTokenizer extends Disposable { } private _endSync(): void { + this._isSynced = false; this._worker.acceptRemovedModel(this._model.uri.toString()); }