From 8a5f705e96247921108b42fb27e4f888feb9dd42 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Wed, 3 Jul 2019 16:41:05 +0200 Subject: [PATCH] Extract IModelLinesTokens --- src/vs/editor/common/model/textModel.ts | 37 ++--- src/vs/editor/common/model/textModelTokens.ts | 133 ++++++++++++------ .../test/common/model/model.line.test.ts | 6 +- .../test/common/model/model.modes.test.ts | 17 +-- 4 files changed, 109 insertions(+), 84 deletions(-) diff --git a/src/vs/editor/common/model/textModel.ts b/src/vs/editor/common/model/textModel.ts index 25a47febe22..5f404d807e8 100644 --- a/src/vs/editor/common/model/textModel.ts +++ b/src/vs/editor/common/model/textModel.ts @@ -23,7 +23,7 @@ import { IntervalNode, IntervalTree, getNodeIsInOverviewRuler, recomputeMaxEnd } import { PieceTreeTextBufferBuilder } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBufferBuilder'; import { IModelContentChangedEvent, IModelDecorationsChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent, IModelOptionsChangedEvent, IModelTokensChangedEvent, InternalModelContentChangeEvent, ModelRawChange, ModelRawContentChangedEvent, ModelRawEOLChanged, ModelRawFlush, ModelRawLineChanged, ModelRawLinesDeleted, ModelRawLinesInserted } from 'vs/editor/common/model/textModelEvents'; import { SearchData, SearchParams, TextModelSearch } from 'vs/editor/common/model/textModelSearch'; -import { ModelLinesTokens, ModelTokensChangedEventBuilder } from 'vs/editor/common/model/textModelTokens'; +import { ModelLinesTokens, ModelTokensChangedEventBuilder, IModelLinesTokens, safeTokenize } from 'vs/editor/common/model/textModelTokens'; import { getWordAtText } from 'vs/editor/common/model/wordHelper'; import { IState, LanguageId, LanguageIdentifier, TokenizationRegistry, FormattingOptions } from 'vs/editor/common/modes'; import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry'; @@ -287,7 +287,7 @@ export class TextModel extends Disposable implements model.ITextModel { private readonly _tokenizationListener: IDisposable; private readonly _languageRegistryListener: IDisposable; private _revalidateTokensTimeout: any; - /*private*/_tokens: ModelLinesTokens; + /*private*/_tokens: IModelLinesTokens; //#endregion constructor(source: string | model.ITextBufferFactory, creationOptions: model.ITextModelCreationOptions, languageIdentifier: LanguageIdentifier | null, associatedResource: URI | null = null) { @@ -1345,7 +1345,7 @@ export class TextModel extends Disposable implements model.ITextModel { this._tokens.applyEdits(change.range, eolCount, firstLineLength); } catch (err) { // emergency recovery => reset tokens - this._tokens = new ModelLinesTokens(this._tokens.languageIdentifier, this._tokens.tokenizationSupport); + this._tokens = new ModelLinesTokens(this._languageIdentifier, this._tokens.tokenizationSupport); } this._onDidChangeDecorations.fire(); this._decorationsTree.acceptReplace(change.rangeOffset, change.rangeLength, change.text.length, change.forceMoveMarkers); @@ -1806,7 +1806,7 @@ export class TextModel extends Disposable implements model.ITextModel { } if (newNonWhitespaceIndex < nonWhitespaceColumn) { - initialState = this._tokens._getState(i - 1); + initialState = this._tokens.getState(i - 1); if (initialState) { break; } @@ -1819,33 +1819,18 @@ export class TextModel extends Disposable implements model.ITextModel { initialState = this._tokens.tokenizationSupport.getInitialState(); } - let state = initialState.clone(); + let state = initialState; for (let i = fakeLines.length - 1; i >= 0; i--) { - let r = this._tokens._tokenizeText(this._buffer, fakeLines[i], state); + let r = safeTokenize(this._languageIdentifier, this._tokens.tokenizationSupport, fakeLines[i], state); if (r) { - state = r.endState.clone(); + state = r.endState; } else { - state = initialState.clone(); + state = initialState; } } const eventBuilder = new ModelTokensChangedEventBuilder(); - for (let i = startLineNumber; i <= endLineNumber; i++) { - let text = this.getLineContent(i); - let r = this._tokens._tokenizeText(this._buffer, text, state); - if (r) { - this._tokens._setTokens(this._tokens.languageIdentifier.id, i - 1, text.length, r.tokens); - - // We cannot trust these states/tokens to be valid! - // (see https://github.com/Microsoft/vscode/issues/67607) - this._tokens._invalidateLine(i - 1); - this._tokens._setState(i - 1, state); - state = r.endState.clone(); - eventBuilder.registerChangedTokens(i); - } else { - state = initialState.clone(); - } - } + this._tokens.fakeTokenizeLines(this._buffer, eventBuilder, state, startLineNumber, endLineNumber); const e = eventBuilder.build(); if (e) { @@ -1871,7 +1856,7 @@ export class TextModel extends Disposable implements model.ITextModel { const eventBuilder = new ModelTokensChangedEventBuilder(); - this._tokens._updateTokensUntilLine(this._buffer, eventBuilder, lineNumber); + this._tokens.updateTokensUntilLine(this._buffer, eventBuilder, lineNumber); const e = eventBuilder.build(); if (e) { @@ -1989,7 +1974,7 @@ export class TextModel extends Disposable implements model.ITextModel { break; } - const tokenizedLineNumber = this._tokens._tokenizeOneLine(this._buffer, eventBuilder); + const tokenizedLineNumber = this._tokens.tokenizeOneInvalidLine(this._buffer, eventBuilder); if (tokenizedLineNumber >= toLineNumber) { break; diff --git a/src/vs/editor/common/model/textModelTokens.ts b/src/vs/editor/common/model/textModelTokens.ts index 4dbdda80829..35fac7434f5 100644 --- a/src/vs/editor/common/model/textModelTokens.ts +++ b/src/vs/editor/common/model/textModelTokens.ts @@ -388,7 +388,27 @@ class TokensStore { //#endregion } -export class ModelLinesTokens { +export interface IModelLinesTokens { + readonly tokenizationSupport: ITokenizationSupport | null; + readonly invalidLineStartIndex: number; + + setTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineTextLength: number, tokens: Uint32Array): void; + getTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineText: string): LineTokens; + + isCheapToTokenize(lineNumber: number): boolean; + hasLinesToTokenize(buffer: ITextBuffer): boolean; + getState(lineIndex: number): IState | null; + applyEdits(range: Range, eolCount: number, firstLineLength: number): void; + + tokenizeOneInvalidLine(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder): number; + updateTokensUntilLine(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder, lineNumber: number): void; + fakeTokenizeLines(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder, initialState: IState, startLineNumber: number, endLineNumber: number): void; + + _getAllStates(linesLength: number): (IState | null)[]; + _getAllInvalid(linesLength: number): number[]; +} + +export class ModelLinesTokens implements IModelLinesTokens { public readonly languageIdentifier: LanguageIdentifier; public readonly tokenizationSupport: ITokenizationSupport | null; @@ -427,19 +447,15 @@ export class ModelLinesTokens { return (this.store.invalidLineStartIndex < buffer.getLineCount()); } - _isInvalid(lineIndex: number): boolean { - return this.store.isInvalid(lineIndex); - } - - _getState(lineIndex: number): IState | null { + public getState(lineIndex: number): IState | null { return this.store.getState(lineIndex); } - _setTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineTextLength: number, tokens: Uint32Array): void { + public setTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineTextLength: number, tokens: Uint32Array): void { this.store.setTokens(topLevelLanguageId, lineIndex, lineTextLength, tokens); } - _setState(lineIndex: number, state: IState): void { + private _setState(lineIndex: number, state: IState): void { this.store.setState(lineIndex, state); } @@ -449,7 +465,7 @@ export class ModelLinesTokens { this.store.applyEdits(range, eolCount, firstLineLength); } - _invalidateLine(lineIndex: number): void { + private _invalidateLine(lineIndex: number): void { this.store.invalidateLine(lineIndex); } @@ -457,33 +473,16 @@ export class ModelLinesTokens { //#region Tokenization - public _tokenizeOneLine(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder): number { + public tokenizeOneInvalidLine(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder): number { if (!this.hasLinesToTokenize(buffer)) { return buffer.getLineCount() + 1; } const lineNumber = this.store.invalidLineStartIndex + 1; - this._updateTokensUntilLine(buffer, eventBuilder, lineNumber); + this.updateTokensUntilLine(buffer, eventBuilder, lineNumber); return lineNumber; } - public _tokenizeText(buffer: ITextBuffer, text: string, state: IState): TokenizationResult2 { - let r: TokenizationResult2 | null = null; - - if (this.tokenizationSupport) { - try { - r = this.tokenizationSupport.tokenize2(text, state, 0); - } catch (e) { - onUnexpectedError(e); - } - } - - if (!r) { - r = nullTokenize2(this.languageIdentifier.id, text, state, 0); - } - return r; - } - - public _updateTokensUntilLine(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder, lineNumber: number): void { + public updateTokensUntilLine(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder, lineNumber: number): void { if (!this.tokenizationSupport) { this.store._invalidLineStartIndex = buffer.getLineCount(); return; @@ -495,28 +494,76 @@ export class ModelLinesTokens { // Validate all states up to and including endLineIndex for (let lineIndex = this.store.invalidLineStartIndex; lineIndex <= endLineIndex; lineIndex++) { const text = buffer.getLineContent(lineIndex + 1); - const lineStartState = this._getState(lineIndex); - - let r: TokenizationResult2 | null = null; - - try { - // Tokenize only the first X characters - let freshState = lineStartState!.clone(); - r = this.tokenizationSupport.tokenize2(text, freshState, 0); - } catch (e) { - onUnexpectedError(e); - } + const lineStartState = this.getState(lineIndex); - if (!r) { - r = nullTokenize2(this.languageIdentifier.id, text, lineStartState, 0); - } + const r = safeTokenize(this.languageIdentifier, this.tokenizationSupport, text, lineStartState!); this.store.setGoodTokens(this.languageIdentifier.id, linesLength, lineIndex, text, r); eventBuilder.registerChangedTokens(lineIndex + 1); lineIndex = this.store.invalidLineStartIndex - 1; // -1 because the outer loop increments it } } + public fakeTokenizeLines(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder, initialState: IState, startLineNumber: number, endLineNumber: number): void { + if (!this.tokenizationSupport) { + return; + } + + let state = initialState; + for (let i = startLineNumber; i <= endLineNumber; i++) { + let text = buffer.getLineContent(i); + let r = safeTokenize(this.languageIdentifier, this.tokenizationSupport, text, state); + if (r) { + this.setTokens(this.languageIdentifier.id, i - 1, text.length, r.tokens); + + // We cannot trust these states/tokens to be valid! + // (see https://github.com/Microsoft/vscode/issues/67607) + this._invalidateLine(i - 1); + this._setState(i - 1, state); + state = r.endState; + eventBuilder.registerChangedTokens(i); + } else { + state = initialState; + } + } + } + // #endregion + + _getAllStates(linesLength: number): (IState | null)[] { + const r: (IState | null)[] = []; + for (let i = 0; i < linesLength; i++) { + r[i] = this.getState(i); + } + r[linesLength] = this.store._lastState; + return r; + } + + _getAllInvalid(linesLength: number): number[] { + const r: number[] = []; + for (let i = 0; i < linesLength; i++) { + if (this.store.isInvalid(i)) { + r.push(i); + } + } + return r; + } +} + +export function safeTokenize(languageIdentifier: LanguageIdentifier, tokenizationSupport: ITokenizationSupport | null, text: string, state: IState): TokenizationResult2 { + let r: TokenizationResult2 | null = null; + + if (tokenizationSupport) { + try { + r = tokenizationSupport.tokenize2(text, state.clone(), 0); + } catch (e) { + onUnexpectedError(e); + } + } + + if (!r) { + r = nullTokenize2(languageIdentifier.id, text, state, 0); + } + return r; } export class ModelTokensChangedEventBuilder { diff --git a/src/vs/editor/test/common/model/model.line.test.ts b/src/vs/editor/test/common/model/model.line.test.ts index 19379cd5c2f..3c00d5a72b3 100644 --- a/src/vs/editor/test/common/model/model.line.test.ts +++ b/src/vs/editor/test/common/model/model.line.test.ts @@ -110,7 +110,7 @@ suite('ModelLinesTokens', () => { for (let lineIndex = 0; lineIndex < initial.length; lineIndex++) { const lineTokens = initial[lineIndex].tokens; const lineTextLength = model.getLineMaxColumn(lineIndex + 1) - 1; - model._tokens._setTokens(0, lineIndex, lineTextLength, TestToken.toTokens(lineTokens)); + model._tokens.setTokens(0, lineIndex, lineTextLength, TestToken.toTokens(lineTokens)); } model.applyEdits(edits.map((ed) => ({ @@ -441,14 +441,14 @@ suite('ModelLinesTokens', () => { test('insertion on empty line', () => { const model = new TextModel('some text', TextModel.DEFAULT_CREATION_OPTIONS, new LanguageIdentifier('test', 0)); - model._tokens._setTokens(0, 0, model.getLineMaxColumn(1) - 1, TestToken.toTokens([new TestToken(0, 1)])); + model._tokens.setTokens(0, 0, model.getLineMaxColumn(1) - 1, TestToken.toTokens([new TestToken(0, 1)])); model.applyEdits([{ range: new Range(1, 1, 1, 10), text: '' }]); - model._tokens._setTokens(0, 0, model.getLineMaxColumn(1) - 1, new Uint32Array(0)); + model._tokens.setTokens(0, 0, model.getLineMaxColumn(1) - 1, new Uint32Array(0)); model.applyEdits([{ range: new Range(1, 1, 1, 1), diff --git a/src/vs/editor/test/common/model/model.modes.test.ts b/src/vs/editor/test/common/model/model.modes.test.ts index 61a8bee1f08..4252f69541e 100644 --- a/src/vs/editor/test/common/model/model.modes.test.ts +++ b/src/vs/editor/test/common/model/model.modes.test.ts @@ -180,25 +180,18 @@ suite('Editor Model - Model Modes 2', () => { }; function invalidEqual(model: TextModel, expected: number[]): void { - let actual: number[] = []; - for (let i = 0, len = model.getLineCount(); i < len; i++) { - if (model._tokens._isInvalid(i)) { - actual.push(i); - } - } - assert.deepEqual(actual, expected); + assert.deepEqual(model._tokens._getAllInvalid(model.getLineCount()), expected); } function stateEqual(state: modes.IState, content: string): void { assert.equal((state).prevLineContent, content); } - function statesEqual(model: TextModel, states: string[]): void { - let i, len = states.length - 1; - for (i = 0; i < len; i++) { - stateEqual(model._tokens._getState(i)!, states[i]); + function statesEqual(model: TextModel, expectedStates: string[]): void { + const actualStates = model._tokens._getAllStates(model.getLineCount()); + for (let i = 0, len = expectedStates.length; i < len; i++) { + stateEqual(actualStates[i]!, expectedStates[i]); } - stateEqual(model._tokens.store._lastState!, states[len]); } let thisModel: TextModel; -- GitLab