From 921fde6502feb372c8c45832710444b163e83d8e Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Tue, 20 Dec 2016 10:41:22 +0100 Subject: [PATCH] Reduce usage of TokenizationSupport --- .../common/model/textModelWithTokens.ts | 3 +- src/vs/editor/common/modes.ts | 5 - src/vs/editor/common/modes/supports.ts | 3 - .../test/common/tokenSelectionSupport.test.ts | 4 +- src/vs/editor/test/common/mocks/mockMode.ts | 39 +---- .../test/common/model/model.modes.test.ts | 155 +++++++++--------- .../common/model/textModelWithTokens.test.ts | 16 +- .../common/modes/textToHtmlTokenizer.test.ts | 46 +++--- src/vs/monaco.d.ts | 4 - 9 files changed, 111 insertions(+), 164 deletions(-) diff --git a/src/vs/editor/common/model/textModelWithTokens.ts b/src/vs/editor/common/model/textModelWithTokens.ts index cd8e6404635..3ffa3e17768 100644 --- a/src/vs/editor/common/model/textModelWithTokens.ts +++ b/src/vs/editor/common/model/textModelWithTokens.ts @@ -365,7 +365,8 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke try { // Tokenize only the first X characters - r = this._tokenizationSupport.tokenize(this._lines[lineIndex].text, this._lines[lineIndex].getState(), 0, stopLineTokenizationAfter); + let freshState = this._lines[lineIndex].getState().clone(); + r = this._tokenizationSupport.tokenize(this._lines[lineIndex].text, freshState, 0, stopLineTokenizationAfter); } catch (e) { e.friendlyMessage = TextModelWithTokens.MODE_TOKENIZATION_FAILED_MSG; onUnexpectedError(e); diff --git a/src/vs/editor/common/modes.ts b/src/vs/editor/common/modes.ts index 3edd1c8c261..4a85f5f1c4e 100644 --- a/src/vs/editor/common/modes.ts +++ b/src/vs/editor/common/modes.ts @@ -7,7 +7,6 @@ import { MarkedString } from 'vs/base/common/htmlContent'; import { IDisposable } from 'vs/base/common/lifecycle'; import URI from 'vs/base/common/uri'; -import { TPromise } from 'vs/base/common/winjs.base'; import { IFilter } from 'vs/base/common/filters'; import * as editorCommon from 'vs/editor/common/editorCommon'; import { ModeTransition } from 'vs/editor/common/core/modeTransition'; @@ -87,10 +86,6 @@ export interface ILineTokens2 { * A pointer will be held to this and the object should not be modified by the tokenizer after the pointer is returned. */ endState: IState2; - /** - * An optional promise to force the model to retokenize this line (e.g. missing information at the point of tokenization) - */ - retokenize?: TPromise; } /** * The state of the tokenizer between two lines. diff --git a/src/vs/editor/common/modes/supports.ts b/src/vs/editor/common/modes/supports.ts index 436433a81e4..05f8ad8af76 100644 --- a/src/vs/editor/common/modes/supports.ts +++ b/src/vs/editor/common/modes/supports.ts @@ -4,7 +4,6 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; -import { TPromise } from 'vs/base/common/winjs.base'; import * as modes from 'vs/editor/common/modes'; import { ModeTransition } from 'vs/editor/common/core/modeTransition'; import { Token } from 'vs/editor/common/core/token'; @@ -17,14 +16,12 @@ export class RawLineTokens implements modes.ILineTokens { modeTransitions: ModeTransition[]; actualStopOffset: number; endState: modes.IState; - retokenize: TPromise; constructor(tokens: Token[], modeTransitions: ModeTransition[], actualStopOffset: number, endState: modes.IState) { this.tokens = tokens; this.modeTransitions = modeTransitions; this.actualStopOffset = actualStopOffset; this.endState = endState; - this.retokenize = null; } } diff --git a/src/vs/editor/contrib/smartSelect/test/common/tokenSelectionSupport.test.ts b/src/vs/editor/contrib/smartSelect/test/common/tokenSelectionSupport.test.ts index ad8a30c2766..1a27b066ea6 100644 --- a/src/vs/editor/contrib/smartSelect/test/common/tokenSelectionSupport.test.ts +++ b/src/vs/editor/contrib/smartSelect/test/common/tokenSelectionSupport.test.ts @@ -10,12 +10,12 @@ import { Range } from 'vs/editor/common/core/range'; import { IMode } from 'vs/editor/common/modes'; import { IndentAction } from 'vs/editor/common/modes/languageConfiguration'; import { TokenSelectionSupport } from 'vs/editor/contrib/smartSelect/common/tokenSelectionSupport'; -import { MockTokenizingMode } from 'vs/editor/test/common/mocks/mockMode'; +import { MockMode } from 'vs/editor/test/common/mocks/mockMode'; import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry'; import { ModelServiceImpl } from 'vs/editor/common/services/modelServiceImpl'; import { TestConfigurationService } from 'vs/platform/configuration/test/common/testConfigurationService'; -class MockJSMode extends MockTokenizingMode { +class MockJSMode extends MockMode { constructor() { super('mock-js'); diff --git a/src/vs/editor/test/common/mocks/mockMode.ts b/src/vs/editor/test/common/mocks/mockMode.ts index 41295249f64..cca5ca0478d 100644 --- a/src/vs/editor/test/common/mocks/mockMode.ts +++ b/src/vs/editor/test/common/mocks/mockMode.ts @@ -4,10 +4,7 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; -import { IMode, IState, TokenizationRegistry } from 'vs/editor/common/modes'; -import { AbstractState, ITokenizationResult } from 'vs/editor/common/modes/abstractState'; -import { TokenizationSupport } from 'vs/editor/common/modes/supports/tokenizationSupport'; -import { LineStream } from 'vs/editor/common/modes/lineStream'; +import { IMode } from 'vs/editor/common/modes'; let instanceCount = 0; function generateMockModeId(): string { @@ -28,37 +25,3 @@ export class MockMode implements IMode { return this._id; } } - -export class StateForMockTokenizingMode extends AbstractState { - - private _tokenType: string; - - constructor(modeId: string, tokenType: string) { - super(modeId); - this._tokenType = tokenType; - } - - public makeClone(): StateForMockTokenizingMode { - return this; - } - - public equals(other: IState): boolean { - return true; - } - - public tokenize(stream: LineStream): ITokenizationResult { - stream.advanceToEOS(); - return { type: this._tokenType }; - } -} - -export class MockTokenizingMode extends MockMode { - - constructor(tokenType: string) { - super(); - - TokenizationRegistry.register(this.getId(), new TokenizationSupport(null, this.getId(), { - getInitialState: () => new StateForMockTokenizingMode(this.getId(), tokenType) - }, false)); - } -} diff --git a/src/vs/editor/test/common/model/model.modes.test.ts b/src/vs/editor/test/common/model/model.modes.test.ts index 15d3f85d9a6..c4193d26b7b 100644 --- a/src/vs/editor/test/common/model/model.modes.test.ts +++ b/src/vs/editor/test/common/model/model.modes.test.ts @@ -9,20 +9,11 @@ import { EditOperation } from 'vs/editor/common/core/editOperation'; import { Position } from 'vs/editor/common/core/position'; import { Range } from 'vs/editor/common/core/range'; import { Model } from 'vs/editor/common/model/model'; -import { AbstractState, ITokenizationResult } from 'vs/editor/common/modes/abstractState'; import * as modes from 'vs/editor/common/modes'; -import { TokenizationSupport } from 'vs/editor/common/modes/supports/tokenizationSupport'; -import { LineStream } from 'vs/editor/common/modes/lineStream'; +import { Token } from 'vs/editor/common/core/token'; // --------- utils -var LINE1 = '1'; -var LINE2 = '2'; -var LINE3 = '3'; -var LINE4 = '4'; -var LINE5 = '5'; - - suite('Editor Model - Model Modes 1', () => { const LANGUAGE_ID = 'modelModeTest1'; @@ -32,39 +23,40 @@ suite('Editor Model - Model Modes 1', () => { }; let thisModel: Model; - class ModelState1 extends AbstractState { - public makeClone(): ModelState1 { - return this; - } - public equals(other: modes.IState): boolean { - return this === other; - } - public tokenize(stream: LineStream): ITokenizationResult { - let chr = stream.peek(); - stream.advance(1); - calledState.calledFor.push(chr); - stream.advanceToEOS(); - return { type: '' }; - } - } - function checkAndClear(calledState: { calledFor: string[] }, arr: string[]) { assert.deepEqual(calledState.calledFor, arr); calledState.calledFor = []; } - modes.TokenizationRegistry.register(LANGUAGE_ID, new TokenizationSupport(null, LANGUAGE_ID, { - getInitialState: () => new ModelState1(LANGUAGE_ID) - }, false)); + class ModelState1 implements modes.IState { + clone(): modes.IState { return this; } + equals(other: modes.IState): boolean { return this === other; } + getModeId(): string { return LANGUAGE_ID; } + getStateData(): modes.IState { throw new Error('Not implemented'); } + setStateData(state: modes.IState): void { throw new Error('Not implemented'); } + } + + modes.TokenizationRegistry.register(LANGUAGE_ID, { + getInitialState: () => new ModelState1(), + tokenize: (line: string, state: modes.IState): modes.ILineTokens => { + calledState.calledFor.push(line.charAt(0)); + return { + tokens: [new Token(0, '')], + actualStopOffset: line.length, + endState: state, + modeTransitions: null + }; + } + }); setup(() => { calledState.calledFor = []; var text = - LINE1 + '\r\n' + - LINE2 + '\n' + - LINE3 + '\n' + - LINE4 + '\r\n' + - LINE5; + '1\r\n' + + '2\n' + + '3\n' + + '4\r\n' + + '5'; thisModel = Model.createFromString(text, undefined, LANGUAGE_ID); }); @@ -168,38 +160,45 @@ suite('Editor Model - Model Modes 1', () => { }); }); - - suite('Editor Model - Model Modes 2', () => { const LANGUAGE_ID = 'modelModeTest2'; - class ModelState2 extends AbstractState { - - private prevLineContent: string; + class ModelState2 implements modes.IState { + prevLineContent: string; - constructor(modeId: string, prevLineContent: string) { - super(modeId); + constructor(prevLineContent: string) { this.prevLineContent = prevLineContent; } - public makeClone(): ModelState2 { - return new ModelState2(this.getModeId(), this.prevLineContent); + clone(): modes.IState { + return new ModelState2(this.prevLineContent); } - public equals(other: modes.IState): boolean { - return (other instanceof ModelState2) && (this.prevLineContent === (other).prevLineContent); + equals(other: modes.IState): boolean { + return (other instanceof ModelState2) && other.prevLineContent === this.prevLineContent; } - public tokenize(stream: LineStream): ITokenizationResult { - var line = stream.advanceToEOS(); - this.prevLineContent = line; - return { type: '' }; + getModeId(): string { + return LANGUAGE_ID; } + + getStateData(): modes.IState { throw new Error('Not implemented'); } + setStateData(state: modes.IState): void { throw new Error('Not implemented'); } } - modes.TokenizationRegistry.register(LANGUAGE_ID, new TokenizationSupport(null, LANGUAGE_ID, { - getInitialState: () => new ModelState2(LANGUAGE_ID, '') - }, false)); + + modes.TokenizationRegistry.register(LANGUAGE_ID, { + getInitialState: () => new ModelState2(''), + tokenize: (line: string, state: modes.IState): modes.ILineTokens => { + (state).prevLineContent = line; + return { + tokens: [new Token(0, '')], + actualStopOffset: line.length, + endState: state, + modeTransitions: null + }; + } + }); function invalidEqual(model, indexArray) { var i, len, asHash = {}; @@ -306,39 +305,31 @@ suite('Editor Model - Token Iterator', () => { const LANGUAGE_ID = 'modelModeTestTokenIterator'; - class NState extends AbstractState { - - private n: number; - private allResults: ITokenizationResult[]; - - constructor(modeId: string, n: number) { - super(modeId); - this.n = n; - this.allResults = null; - } - - public makeClone(): NState { - return this; - } - - public equals(other: modes.IState): boolean { - return true; - } + class NState implements modes.IState { + clone(): modes.IState { return this; } + equals(other: modes.IState): boolean { return this === other; } + getModeId(): string { return LANGUAGE_ID; } + getStateData(): modes.IState { throw new Error('Not implemented'); } + setStateData(state: modes.IState): void { throw new Error('Not implemented'); } + } - public tokenize(stream: LineStream): ITokenizationResult { - var ndash = this.n, value = ''; - while (!stream.eos() && ndash > 0) { - let chr = stream.peek(); - stream.advance(1); - value += chr; - ndash--; + modes.TokenizationRegistry.register(LANGUAGE_ID, { + getInitialState: (): modes.IState => new NState(), + tokenize: (line: string, state: modes.IState): modes.ILineTokens => { + let tokens: Token[] = []; + for (let i = 0; i < line.length / 3; i++) { + let from = 3 * i; + let to = from + 3; + tokens.push(new Token(from, 'n-3-' + line.substring(from, to))); } - return { type: 'n-' + (this.n - ndash) + '-' + value }; + return { + tokens: tokens, + actualStopOffset: line.length, + endState: state, + modeTransitions: null + }; } - } - modes.TokenizationRegistry.register(LANGUAGE_ID, new TokenizationSupport(null, LANGUAGE_ID, { - getInitialState: () => new NState(LANGUAGE_ID, 3) - }, false)); + }); var thisModel: Model; diff --git a/src/vs/editor/test/common/model/textModelWithTokens.test.ts b/src/vs/editor/test/common/model/textModelWithTokens.test.ts index 8e94e5e4c1f..2ba27f82243 100644 --- a/src/vs/editor/test/common/model/textModelWithTokens.test.ts +++ b/src/vs/editor/test/common/model/textModelWithTokens.test.ts @@ -7,7 +7,7 @@ import * as assert from 'assert'; import { Model } from 'vs/editor/common/model/model'; import { ViewLineToken } from 'vs/editor/common/core/viewLineToken'; -import { TokenizationRegistry } from 'vs/editor/common/modes'; +import { TokenizationRegistry, IState } from 'vs/editor/common/modes'; import { CharacterPair } from 'vs/editor/common/modes/languageConfiguration'; import { MockMode } from 'vs/editor/test/common/mocks/mockMode'; import { Token } from 'vs/editor/common/core/token'; @@ -260,21 +260,27 @@ suite('TextModelWithTokens regression tests', () => { } let _tokenId = 0; + class IndicisiveModeState implements IState { + clone(): IState { return this; } + equals(other: IState): boolean { return true; } + getModeId(): string { throw new Error('Not implemented'); } + getStateData(): IState { throw new Error('Not implemented'); } + setStateData(state: IState): void { throw new Error('Not implemented'); } + } class IndicisiveMode extends MockMode { constructor() { super(); TokenizationRegistry.register(this.getId(), { getInitialState: () => { - return null; + return new IndicisiveModeState(); }, tokenize: (line, state, offsetDelta, stopAtOffset) => { let myId = ++_tokenId; return { tokens: [new Token(0, 'custom.' + myId)], actualStopOffset: line.length, - endState: null, - modeTransitions: [], - retokenize: null + endState: state, + modeTransitions: [] }; } }); diff --git a/src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts b/src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts index 51e54052d9f..4df0382c329 100644 --- a/src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts +++ b/src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts @@ -5,12 +5,10 @@ 'use strict'; import * as assert from 'assert'; -import { TokenizationRegistry } from 'vs/editor/common/modes'; -import { AbstractState, ITokenizationResult } from 'vs/editor/common/modes/abstractState'; -import { TokenizationSupport } from 'vs/editor/common/modes/supports/tokenizationSupport'; +import { TokenizationRegistry, IState, ILineTokens } from 'vs/editor/common/modes'; import { tokenizeToHtmlContent } from 'vs/editor/common/modes/textToHtmlTokenizer'; import { MockMode } from 'vs/editor/test/common/mocks/mockMode'; -import { LineStream } from 'vs/editor/common/modes/lineStream'; +import { Token } from 'vs/editor/common/core/token'; suite('Editor Modes - textToHtmlTokenizer', () => { test('TextToHtmlTokenizer', () => { @@ -58,28 +56,28 @@ suite('Editor Modes - textToHtmlTokenizer', () => { }); -class State extends AbstractState { - - constructor(modeId: string) { - super(modeId); - } - - public makeClone(): AbstractState { - return new State(this.getModeId()); - } - - public tokenize(stream: LineStream): ITokenizationResult { - let chr = stream.peek(); - stream.advance(1); - return { type: chr === '.' ? '' : 'text' }; - } -} - class Mode extends MockMode { constructor() { super(); - TokenizationRegistry.register(this.getId(), new TokenizationSupport(null, this.getId(), { - getInitialState: () => new State(this.getId()) - }, false)); + TokenizationRegistry.register(this.getId(), { + getInitialState: (): IState => null, + tokenize: (line: string, state: IState): ILineTokens => { + let tokens: Token[] = []; + for (let i = 0; i < line.length; i++) { + let chr = line.charAt(i); + let type = chr === '.' ? '' : 'text'; + if (tokens.length > 0 && tokens[tokens.length - 1].type === type) { + continue; + } + tokens.push(new Token(i, type)); + } + return { + tokens: tokens, + actualStopOffset: -1, + endState: null, + modeTransitions: null + }; + } + }); } } diff --git a/src/vs/monaco.d.ts b/src/vs/monaco.d.ts index a3f8dcc2099..ba7d847ed8e 100644 --- a/src/vs/monaco.d.ts +++ b/src/vs/monaco.d.ts @@ -4259,10 +4259,6 @@ declare module monaco.languages { * A pointer will be held to this and the object should not be modified by the tokenizer after the pointer is returned. */ endState: IState; - /** - * An optional promise to force the model to retokenize this line (e.g. missing information at the point of tokenization) - */ - retokenize?: Promise; } /** -- GitLab