Reduce usage of TokenizationSupport

src/vs/editor/common/model/textModelWithTokens.ts

@@ -365,7 +365,8 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 
 			try {
 				// Tokenize only the first X characters
-				r = this._tokenizationSupport.tokenize(this._lines[lineIndex].text, this._lines[lineIndex].getState(), 0, stopLineTokenizationAfter);
+				let freshState = this._lines[lineIndex].getState().clone();
+				r = this._tokenizationSupport.tokenize(this._lines[lineIndex].text, freshState, 0, stopLineTokenizationAfter);
 			} catch (e) {
 				e.friendlyMessage = TextModelWithTokens.MODE_TOKENIZATION_FAILED_MSG;
 				onUnexpectedError(e);

src/vs/editor/common/modes.ts

@@ -7,7 +7,6 @@
 import { MarkedString } from 'vs/base/common/htmlContent';
 import { IDisposable } from 'vs/base/common/lifecycle';
 import URI from 'vs/base/common/uri';
-import { TPromise } from 'vs/base/common/winjs.base';
 import { IFilter } from 'vs/base/common/filters';
 import * as editorCommon from 'vs/editor/common/editorCommon';
 import { ModeTransition } from 'vs/editor/common/core/modeTransition';
@@ -87,10 +86,6 @@ export interface ILineTokens2 {
 	 * A pointer will be held to this and the object should not be modified by the tokenizer after the pointer is returned.
 	 */
 	endState: IState2;
-	/**
-	 * An optional promise to force the model to retokenize this line (e.g. missing information at the point of tokenization)
-	 */
-	retokenize?: TPromise<void>;
 }
 /**
  * The state of the tokenizer between two lines.

src/vs/editor/common/modes/supports.ts

@@ -4,7 +4,6 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
-import { TPromise } from 'vs/base/common/winjs.base';
 import * as modes from 'vs/editor/common/modes';
 import { ModeTransition } from 'vs/editor/common/core/modeTransition';
 import { Token } from 'vs/editor/common/core/token';
@@ -17,14 +16,12 @@ export class RawLineTokens implements modes.ILineTokens {
 	modeTransitions: ModeTransition[];
 	actualStopOffset: number;
 	endState: modes.IState;
-	retokenize: TPromise<void>;
 
 	constructor(tokens: Token[], modeTransitions: ModeTransition[], actualStopOffset: number, endState: modes.IState) {
 		this.tokens = tokens;
 		this.modeTransitions = modeTransitions;
 		this.actualStopOffset = actualStopOffset;
 		this.endState = endState;
-		this.retokenize = null;
 	}
 }
 

src/vs/editor/contrib/smartSelect/test/common/tokenSelectionSupport.test.ts

@@ -10,12 +10,12 @@ import { Range } from 'vs/editor/common/core/range';
 import { IMode } from 'vs/editor/common/modes';
 import { IndentAction } from 'vs/editor/common/modes/languageConfiguration';
 import { TokenSelectionSupport } from 'vs/editor/contrib/smartSelect/common/tokenSelectionSupport';
-import { MockTokenizingMode } from 'vs/editor/test/common/mocks/mockMode';
+import { MockMode } from 'vs/editor/test/common/mocks/mockMode';
 import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
 import { ModelServiceImpl } from 'vs/editor/common/services/modelServiceImpl';
 import { TestConfigurationService } from 'vs/platform/configuration/test/common/testConfigurationService';
 
-class MockJSMode extends MockTokenizingMode {
+class MockJSMode extends MockMode {
 
 	constructor() {
 		super('mock-js');

src/vs/editor/test/common/mocks/mockMode.ts

@@ -4,10 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
-import { IMode, IState, TokenizationRegistry } from 'vs/editor/common/modes';
-import { AbstractState, ITokenizationResult } from 'vs/editor/common/modes/abstractState';
-import { TokenizationSupport } from 'vs/editor/common/modes/supports/tokenizationSupport';
-import { LineStream } from 'vs/editor/common/modes/lineStream';
+import { IMode } from 'vs/editor/common/modes';
 
 let instanceCount = 0;
 function generateMockModeId(): string {
@@ -28,37 +25,3 @@ export class MockMode implements IMode {
 		return this._id;
 	}
 }
-
-export class StateForMockTokenizingMode extends AbstractState {
-
-	private _tokenType: string;
-
-	constructor(modeId: string, tokenType: string) {
-		super(modeId);
-		this._tokenType = tokenType;
-	}
-
-	public makeClone(): StateForMockTokenizingMode {
-		return this;
-	}
-
-	public equals(other: IState): boolean {
-		return true;
-	}
-
-	public tokenize(stream: LineStream): ITokenizationResult {
-		stream.advanceToEOS();
-		return { type: this._tokenType };
-	}
-}
-
-export class MockTokenizingMode extends MockMode {
-
-	constructor(tokenType: string) {
-		super();
-
-		TokenizationRegistry.register(this.getId(), new TokenizationSupport(null, this.getId(), {
-			getInitialState: () => new StateForMockTokenizingMode(this.getId(), tokenType)
-		}, false));
-	}
-}

src/vs/editor/test/common/model/model.modes.test.ts

@@ -9,20 +9,11 @@ import { EditOperation } from 'vs/editor/common/core/editOperation';
 import { Position } from 'vs/editor/common/core/position';
 import { Range } from 'vs/editor/common/core/range';
 import { Model } from 'vs/editor/common/model/model';
-import { AbstractState, ITokenizationResult } from 'vs/editor/common/modes/abstractState';
 import * as modes from 'vs/editor/common/modes';
-import { TokenizationSupport } from 'vs/editor/common/modes/supports/tokenizationSupport';
-import { LineStream } from 'vs/editor/common/modes/lineStream';
+import { Token } from 'vs/editor/common/core/token';
 
 // --------- utils
 
-var LINE1 = '1';
-var LINE2 = '2';
-var LINE3 = '3';
-var LINE4 = '4';
-var LINE5 = '5';
-
-
 suite('Editor Model - Model Modes 1', () => {
 
 	const LANGUAGE_ID = 'modelModeTest1';
@@ -32,39 +23,40 @@ suite('Editor Model - Model Modes 1', () => {
 	};
 	let thisModel: Model;
 
-	class ModelState1 extends AbstractState {
-		public makeClone(): ModelState1 {
-			return this;
-		}
-		public equals(other: modes.IState): boolean {
-			return this === other;
-		}
-		public tokenize(stream: LineStream): ITokenizationResult {
-			let chr = stream.peek();
-			stream.advance(1);
-			calledState.calledFor.push(chr);
-			stream.advanceToEOS();
-			return { type: '' };
-		}
-	}
-
 	function checkAndClear(calledState: { calledFor: string[] }, arr: string[]) {
 		assert.deepEqual(calledState.calledFor, arr);
 		calledState.calledFor = [];
 	}
 
-	modes.TokenizationRegistry.register(LANGUAGE_ID, new TokenizationSupport(null, LANGUAGE_ID, {
-		getInitialState: () => new ModelState1(LANGUAGE_ID)
-	}, false));
+	class ModelState1 implements modes.IState {
+		clone(): modes.IState { return this; }
+		equals(other: modes.IState): boolean { return this === other; }
+		getModeId(): string { return LANGUAGE_ID; }
+		getStateData(): modes.IState { throw new Error('Not implemented'); }
+		setStateData(state: modes.IState): void { throw new Error('Not implemented'); }
+	}
+
+	modes.TokenizationRegistry.register(LANGUAGE_ID, {
+		getInitialState: () => new ModelState1(),
+		tokenize: (line: string, state: modes.IState): modes.ILineTokens => {
+			calledState.calledFor.push(line.charAt(0));
+			return {
+				tokens: [new Token(0, '')],
+				actualStopOffset: line.length,
+				endState: state,
+				modeTransitions: null
+			};
+		}
+	});
 
 	setup(() => {
 		calledState.calledFor = [];
 		var text =
-			LINE1 + '\r\n' +
-			LINE2 + '\n' +
-			LINE3 + '\n' +
-			LINE4 + '\r\n' +
-			LINE5;
+			'1\r\n' +
+			'2\n' +
+			'3\n' +
+			'4\r\n' +
+			'5';
 		thisModel = Model.createFromString(text, undefined, LANGUAGE_ID);
 	});
 
@@ -168,38 +160,45 @@ suite('Editor Model - Model Modes 1', () => {
 	});
 });
 
-
-
 suite('Editor Model - Model Modes 2', () => {
 
 	const LANGUAGE_ID = 'modelModeTest2';
 
-	class ModelState2 extends AbstractState {
-
-		private prevLineContent: string;
+	class ModelState2 implements modes.IState {
+		prevLineContent: string;
 
-		constructor(modeId: string, prevLineContent: string) {
-			super(modeId);
+		constructor(prevLineContent: string) {
 			this.prevLineContent = prevLineContent;
 		}
 
-		public makeClone(): ModelState2 {
-			return new ModelState2(this.getModeId(), this.prevLineContent);
+		clone(): modes.IState {
+			return new ModelState2(this.prevLineContent);
 		}
 
-		public equals(other: modes.IState): boolean {
-			return (other instanceof ModelState2) && (this.prevLineContent === (<ModelState2>other).prevLineContent);
+		equals(other: modes.IState): boolean {
+			return (other instanceof ModelState2) && other.prevLineContent === this.prevLineContent;
 		}
 
-		public tokenize(stream: LineStream): ITokenizationResult {
-			var line = stream.advanceToEOS();
-			this.prevLineContent = line;
-			return { type: '' };
+		getModeId(): string {
+			return LANGUAGE_ID;
 		}
+
+		getStateData(): modes.IState { throw new Error('Not implemented'); }
+		setStateData(state: modes.IState): void { throw new Error('Not implemented'); }
 	}
-	modes.TokenizationRegistry.register(LANGUAGE_ID, new TokenizationSupport(null, LANGUAGE_ID, {
-		getInitialState: () => new ModelState2(LANGUAGE_ID, '')
-	}, false));
+
+	modes.TokenizationRegistry.register(LANGUAGE_ID, {
+		getInitialState: () => new ModelState2(''),
+		tokenize: (line: string, state: modes.IState): modes.ILineTokens => {
+			(<ModelState2>state).prevLineContent = line;
+			return {
+				tokens: [new Token(0, '')],
+				actualStopOffset: line.length,
+				endState: state,
+				modeTransitions: null
+			};
+		}
+	});
 
 	function invalidEqual(model, indexArray) {
 		var i, len, asHash = {};
@@ -306,39 +305,31 @@ suite('Editor Model - Token Iterator', () => {
 
 	const LANGUAGE_ID = 'modelModeTestTokenIterator';
 
-	class NState extends AbstractState {
-
-		private n: number;
-		private allResults: ITokenizationResult[];
-
-		constructor(modeId: string, n: number) {
-			super(modeId);
-			this.n = n;
-			this.allResults = null;
-		}
-
-		public makeClone(): NState {
-			return this;
-		}
-
-		public equals(other: modes.IState): boolean {
-			return true;
-		}
+	class NState implements modes.IState {
+		clone(): modes.IState { return this; }
+		equals(other: modes.IState): boolean { return this === other; }
+		getModeId(): string { return LANGUAGE_ID; }
+		getStateData(): modes.IState { throw new Error('Not implemented'); }
+		setStateData(state: modes.IState): void { throw new Error('Not implemented'); }
+	}
 
-		public tokenize(stream: LineStream): ITokenizationResult {
-			var ndash = this.n, value = '';
-			while (!stream.eos() && ndash > 0) {
-				let chr = stream.peek();
-				stream.advance(1);
-				value += chr;
-				ndash--;
+	modes.TokenizationRegistry.register(LANGUAGE_ID, {
+		getInitialState: (): modes.IState => new NState(),
+		tokenize: (line: string, state: modes.IState): modes.ILineTokens => {
+			let tokens: Token[] = [];
+			for (let i = 0; i < line.length / 3; i++) {
+				let from = 3 * i;
+				let to = from + 3;
+				tokens.push(new Token(from, 'n-3-' + line.substring(from, to)));
 			}
-			return { type: 'n-' + (this.n - ndash) + '-' + value };
+			return {
+				tokens: tokens,
+				actualStopOffset: line.length,
+				endState: state,
+				modeTransitions: null
+			};
 		}
-	}
-	modes.TokenizationRegistry.register(LANGUAGE_ID, new TokenizationSupport(null, LANGUAGE_ID, {
-		getInitialState: () => new NState(LANGUAGE_ID, 3)
-	}, false));
+	});
 
 	var thisModel: Model;
 

src/vs/editor/test/common/model/textModelWithTokens.test.ts

@@ -7,7 +7,7 @@
 import * as assert from 'assert';
 import { Model } from 'vs/editor/common/model/model';
 import { ViewLineToken } from 'vs/editor/common/core/viewLineToken';
-import { TokenizationRegistry } from 'vs/editor/common/modes';
+import { TokenizationRegistry, IState } from 'vs/editor/common/modes';
 import { CharacterPair } from 'vs/editor/common/modes/languageConfiguration';
 import { MockMode } from 'vs/editor/test/common/mocks/mockMode';
 import { Token } from 'vs/editor/common/core/token';
@@ -260,21 +260,27 @@ suite('TextModelWithTokens regression tests', () => {
 		}
 
 		let _tokenId = 0;
+		class IndicisiveModeState implements IState {
+			clone(): IState { return this; }
+			equals(other: IState): boolean { return true; }
+			getModeId(): string { throw new Error('Not implemented'); }
+			getStateData(): IState { throw new Error('Not implemented'); }
+			setStateData(state: IState): void { throw new Error('Not implemented'); }
+		}
 		class IndicisiveMode extends MockMode {
 			constructor() {
 				super();
 				TokenizationRegistry.register(this.getId(), {
 					getInitialState: () => {
-						return null;
+						return new IndicisiveModeState();
 					},
 					tokenize: (line, state, offsetDelta, stopAtOffset) => {
 						let myId = ++_tokenId;
 						return {
 							tokens: [new Token(0, 'custom.' + myId)],
 							actualStopOffset: line.length,
-							endState: null,
-							modeTransitions: [],
-							retokenize: null
+							endState: state,
+							modeTransitions: []
 						};
 					}
 				});

src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts

@@ -5,12 +5,10 @@
 'use strict';
 
 import * as assert from 'assert';
-import { TokenizationRegistry } from 'vs/editor/common/modes';
-import { AbstractState, ITokenizationResult } from 'vs/editor/common/modes/abstractState';
-import { TokenizationSupport } from 'vs/editor/common/modes/supports/tokenizationSupport';
+import { TokenizationRegistry, IState, ILineTokens } from 'vs/editor/common/modes';
 import { tokenizeToHtmlContent } from 'vs/editor/common/modes/textToHtmlTokenizer';
 import { MockMode } from 'vs/editor/test/common/mocks/mockMode';
-import { LineStream } from 'vs/editor/common/modes/lineStream';
+import { Token } from 'vs/editor/common/core/token';
 
 suite('Editor Modes - textToHtmlTokenizer', () => {
 	test('TextToHtmlTokenizer', () => {
@@ -58,28 +56,28 @@ suite('Editor Modes - textToHtmlTokenizer', () => {
 
 });
 
-class State extends AbstractState {
-
-	constructor(modeId: string) {
-		super(modeId);
-	}
-
-	public makeClone(): AbstractState {
-		return new State(this.getModeId());
-	}
-
-	public tokenize(stream: LineStream): ITokenizationResult {
-		let chr = stream.peek();
-		stream.advance(1);
-		return { type: chr === '.' ? '' : 'text' };
-	}
-}
-
 class Mode extends MockMode {
 	constructor() {
 		super();
-		TokenizationRegistry.register(this.getId(), new TokenizationSupport(null, this.getId(), {
-			getInitialState: () => new State(this.getId())
-		}, false));
+		TokenizationRegistry.register(this.getId(), {
+			getInitialState: (): IState => null,
+			tokenize: (line: string, state: IState): ILineTokens => {
+				let tokens: Token[] = [];
+				for (let i = 0; i < line.length; i++) {
+					let chr = line.charAt(i);
+					let type = chr === '.' ? '' : 'text';
+					if (tokens.length > 0 && tokens[tokens.length - 1].type === type) {
+						continue;
+					}
+					tokens.push(new Token(i, type));
+				}
+				return {
+					tokens: tokens,
+					actualStopOffset: -1,
+					endState: null,
+					modeTransitions: null
+				};
+			}
+		});
 	}
 }

src/vs/monaco.d.ts

@@ -4259,10 +4259,6 @@ declare module monaco.languages {
          * A pointer will be held to this and the object should not be modified by the tokenizer after the pointer is returned.
          */
         endState: IState;
-        /**
-         * An optional promise to force the model to retokenize this line (e.g. missing information at the point of tokenization)
-         */
-        retokenize?: Promise<void>;
     }
 
     /**