Move tokenize down from IState to AbstractState

src/vs/editor/common/modes.ts

@@ -18,15 +18,6 @@ import {Position} from 'vs/editor/common/core/position';
 import {Range} from 'vs/editor/common/core/range';
 import Event, {Emitter} from 'vs/base/common/event';
 
-/**
- * @internal
- */
-export interface ITokenizationResult {
-	type?:string;
-	dontMergeWithPrev?:boolean;
-	nextState?:IState;
-}
-
 /**
  * @internal
  */
@@ -34,7 +25,6 @@ export interface IState {
 	clone():IState;
 	equals(other:IState):boolean;
 	getModeId():string;
-	tokenize(stream:IStream):ITokenizationResult;
 	getStateData(): IState;
 	setStateData(state:IState):void;
 }

src/vs/editor/common/modes/TMState.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
-import {IState, ITokenizationResult} from 'vs/editor/common/modes';
+import {IState} from 'vs/editor/common/modes';
 import {AbstractState} from 'vs/editor/common/modes/abstractState';
 import {StackElement} from 'vscode-textmate';
 
@@ -50,10 +50,6 @@ export class TMState implements IState {
 		return this._modeId;
 	}
 
-	public tokenize(stream:any):ITokenizationResult {
-		throw new Error();
-	}
-
 	public getStateData():IState {
 		return this._parentEmbedderState;
 	}

src/vs/editor/common/modes/abstractState.ts

@@ -4,10 +4,21 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
-import {IState, IStream, ITokenizationResult} from 'vs/editor/common/modes';
+import {IState, IStream} from 'vs/editor/common/modes';
+
+/**
+ * @internal
+ */
+export interface ITokenizationResult {
+	type?:string;
+	dontMergeWithPrev?:boolean;
+	nextState?:AbstractState;
+}
 
 export abstract class AbstractState implements IState {
 
+	_abstractStateBrand: void;
+
 	private modeId:string;
 	private stateData:IState;
 

src/vs/editor/common/modes/monarch/monarchLexer.ts

@@ -10,7 +10,7 @@
  */
 
 import * as modes from 'vs/editor/common/modes';
-import {AbstractState} from 'vs/editor/common/modes/abstractState';
+import {AbstractState, ITokenizationResult} from 'vs/editor/common/modes/abstractState';
 import {LineStream} from 'vs/editor/common/modes/lineStream';
 import * as monarchCommon from 'vs/editor/common/modes/monarch/monarchCommon';
 import {IModeLocator, TokenizationSupport} from 'vs/editor/common/modes/supports/tokenizationSupport';
@@ -94,7 +94,7 @@ export class MonarchLexer extends AbstractState {
 	 * TODO: there are many optimizations possible here for the common cases
 	 * but for now I concentrated on functionality and correctness.
 	 */
-	public tokenize(stream: modes.IStream, noConsumeIsOk?: boolean): modes.ITokenizationResult {
+	public tokenize(stream: modes.IStream, noConsumeIsOk?: boolean): ITokenizationResult {
 		var stackLen0 = this.stack.length;  // these are saved to check progress
 		var groupLen0 = 0;
 		var state: string = this.stack[0];  // the current state
@@ -388,7 +388,7 @@ function findBracket(lexer: monarchCommon.ILexer, matched: string) {
 
 export function createTokenizationSupport(_modeService:IModeService, modeId:string, lexer: monarchCommon.ILexer): modes.ITokenizationSupport {
 	return new TokenizationSupport(_modeService, modeId, {
-		getInitialState: (): modes.IState => {
+		getInitialState: (): AbstractState => {
 			return new MonarchLexer(modeId, _modeService, lexer);
 		},
 

src/vs/editor/common/modes/nullMode.ts

@@ -4,9 +4,10 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
-import {IState, IStream, ITokenizationResult, ILineTokens} from 'vs/editor/common/modes';
+import {IState, IStream, ILineTokens} from 'vs/editor/common/modes';
 import {ModeTransition} from 'vs/editor/common/core/modeTransition';
 import {Token} from 'vs/editor/common/core/token';
+import {ITokenizationResult} from 'vs/editor/common/modes/abstractState';
 
 export class NullState implements IState {
 

src/vs/editor/common/modes/supports/tokenizationSupport.ts

@@ -11,6 +11,7 @@ import {NullState, nullTokenize, NULL_MODE_ID} from 'vs/editor/common/modes/null
 import {Token} from 'vs/editor/common/core/token';
 import {ModeTransition} from 'vs/editor/common/core/modeTransition';
 import {IModeService} from 'vs/editor/common/services/modeService';
+import {AbstractState, ITokenizationResult} from 'vs/editor/common/modes/abstractState';
 
 export interface ILeavingNestedModeData {
 	/**
@@ -26,7 +27,7 @@ export interface ILeavingNestedModeData {
 	/**
 	 * The state that will be used for continuing tokenization by the parent mode after the nested mode
 	 */
-	stateAfterNestedMode: modes.IState;
+	stateAfterNestedMode: AbstractState;
 }
 
 export interface IModeLocator {
@@ -35,11 +36,11 @@ export interface IModeLocator {
 
 export interface ITokenizationCustomization {
 
-	getInitialState():modes.IState;
+	getInitialState():AbstractState;
 
-	enterNestedMode?: (state:modes.IState) => boolean;
+	enterNestedMode?: (state:AbstractState) => boolean;
 
-	getNestedMode?: (state:modes.IState, locator:IModeLocator) => modes.IMode;
+	getNestedMode?: (state:AbstractState, locator:IModeLocator) => modes.IMode;
 
 	/**
 	 * Return null if the line does not leave the nested mode
@@ -109,7 +110,7 @@ export class TokenizationSupport implements modes.ITokenizationSupport, IDisposa
 		if (state.getModeId() !== this._modeId) {
 			return this._nestedTokenize(line, state, deltaOffset, stopAtOffset, [], []);
 		} else {
-			return this._myTokenize(line, state, deltaOffset, stopAtOffset, [], []);
+			return this._myTokenize(line, <AbstractState>state, deltaOffset, stopAtOffset, [], []);
 		}
 	}
 
@@ -178,9 +179,9 @@ export class TokenizationSupport implements modes.ITokenizationSupport, IDisposa
 	 * Precondition is: state.getMode() === this
 	 * This means we are in the current mode when parsing starts on this line.
 	 */
-	private _myTokenize(buffer:string, myState:modes.IState, deltaOffset:number, stopAtOffset:number, prependTokens:Token[], prependModeTransitions:ModeTransition[]):modes.ILineTokens {
+	private _myTokenize(buffer:string, myState:AbstractState, deltaOffset:number, stopAtOffset:number, prependTokens:Token[], prependModeTransitions:ModeTransition[]):modes.ILineTokens {
 		let lineStream = new LineStream(buffer);
-		let tokenResult:modes.ITokenizationResult, beforeTokenizeStreamPos:number;
+		let tokenResult:ITokenizationResult, beforeTokenizeStreamPos:number;
 		let previousType:string = null;
 
 		myState = myState.clone();
@@ -228,7 +229,12 @@ export class TokenizationSupport implements modes.ITokenizationSupport, IDisposa
 						return result;
 					} else {
 						// Transition to the nested mode state
-						myState = nestedModeState;
+						return {
+							tokens: prependTokens,
+							actualStopOffset: lineStream.pos() + deltaOffset,
+							modeTransitions: prependModeTransitions,
+							endState: nestedModeState
+						};
 					}
 				}
 			}
@@ -251,7 +257,7 @@ export class TokenizationSupport implements modes.ITokenizationSupport, IDisposa
 		return result;
 	}
 
-	private _enterNestedMode(state:modes.IState): boolean {
+	private _enterNestedMode(state:AbstractState): boolean {
 		if (this.defaults.enterNestedMode) {
 			return false;
 		}
@@ -259,7 +265,7 @@ export class TokenizationSupport implements modes.ITokenizationSupport, IDisposa
 
 	}
 
-	private _getNestedMode(state:modes.IState): modes.IMode {
+	private _getNestedMode(state:AbstractState): modes.IMode {
 		if (this.defaults.getNestedMode) {
 			return null;
 		}
@@ -291,7 +297,7 @@ export class TokenizationSupport implements modes.ITokenizationSupport, IDisposa
 		return this.customization.getNestedMode(state, locator);
 	}
 
-	private _getNestedModeInitialState(state:modes.IState): modes.IState {
+	private _getNestedModeInitialState(state:AbstractState): modes.IState {
 		let nestedMode = this._getNestedMode(state);
 		if (nestedMode) {
 			let tokenizationSupport = modes.TokenizationRegistry.get(nestedMode.getId());

src/vs/editor/common/services/modelServiceImpl.ts

@@ -26,13 +26,6 @@ import {DEFAULT_INDENTATION, DEFAULT_TRIM_AUTO_WHITESPACE} from 'vs/editor/commo
 import {IMessageService} from 'vs/platform/message/common/message';
 import {PLAINTEXT_MODE_ID} from 'vs/editor/common/modes/modesRegistry';
 
-export interface IRawModelData {
-	url: URI;
-	versionId: number;
-	value: editorCommon.IRawText;
-	modeId: string;
-}
-
 function MODEL_ID(resource: URI): string {
 	return resource.toString();
 }

src/vs/editor/test/common/mocks/mockMode.ts

@@ -4,8 +4,8 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
-import {IMode, IState, IStream, ITokenizationResult, TokenizationRegistry} from 'vs/editor/common/modes';
-import {AbstractState} from 'vs/editor/common/modes/abstractState';
+import {IMode, IState, IStream, TokenizationRegistry} from 'vs/editor/common/modes';
+import {AbstractState, ITokenizationResult} from 'vs/editor/common/modes/abstractState';
 import {TokenizationSupport} from 'vs/editor/common/modes/supports/tokenizationSupport';
 
 let instanceCount = 0;

src/vs/editor/test/common/model/model.modes.test.ts

@@ -9,7 +9,7 @@ import {EditOperation} from 'vs/editor/common/core/editOperation';
 import {Position} from 'vs/editor/common/core/position';
 import {Range} from 'vs/editor/common/core/range';
 import {Model} from 'vs/editor/common/model/model';
-import {AbstractState} from 'vs/editor/common/modes/abstractState';
+import {AbstractState, ITokenizationResult} from 'vs/editor/common/modes/abstractState';
 import * as modes from 'vs/editor/common/modes';
 import {TokenizationSupport} from 'vs/editor/common/modes/supports/tokenizationSupport';
 
@@ -38,7 +38,7 @@ suite('Editor Model - Model Modes 1', () => {
 		public equals(other: modes.IState): boolean {
 			return this === other;
 		}
-		public tokenize(stream:modes.IStream): modes.ITokenizationResult {
+		public tokenize(stream:modes.IStream): ITokenizationResult {
 			calledState.calledFor.push(stream.next());
 			stream.advanceToEOS();
 			return { type: '' };
@@ -188,7 +188,7 @@ suite('Editor Model - Model Modes 2', () => {
 			return (other instanceof ModelState2) && (this.prevLineContent === (<ModelState2>other).prevLineContent);
 		}
 
-		public tokenize(stream:modes.IStream):modes.ITokenizationResult {
+		public tokenize(stream:modes.IStream):ITokenizationResult {
 			var line= '';
 			while (!stream.eos()) {
 				line+= stream.next();
@@ -309,7 +309,7 @@ suite('Editor Model - Token Iterator', () => {
 	class NState extends AbstractState {
 
 		private n:number;
-		private allResults:modes.ITokenizationResult[];
+		private allResults:ITokenizationResult[];
 
 		constructor(modeId:string, n:number) {
 			super(modeId);
@@ -325,7 +325,7 @@ suite('Editor Model - Token Iterator', () => {
 			return true;
 		}
 
-		public tokenize(stream:modes.IStream):modes.ITokenizationResult {
+		public tokenize(stream:modes.IStream):ITokenizationResult {
 			var ndash = this.n, value = '';
 			while(!stream.eos() && ndash > 0) {
 				value += stream.next();

src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts

@@ -5,8 +5,8 @@
 'use strict';
 
 import * as assert from 'assert';
-import {IStream, ITokenizationResult, TokenizationRegistry} from 'vs/editor/common/modes';
-import {AbstractState} from 'vs/editor/common/modes/abstractState';
+import {IStream, TokenizationRegistry} from 'vs/editor/common/modes';
+import {AbstractState, ITokenizationResult} from 'vs/editor/common/modes/abstractState';
 import {TokenizationSupport} from 'vs/editor/common/modes/supports/tokenizationSupport';
 import {tokenizeToHtmlContent} from 'vs/editor/common/modes/textToHtmlTokenizer';
 import {MockMode} from 'vs/editor/test/common/mocks/mockMode';

src/vs/editor/test/common/modes/tokenization.test.ts

@@ -6,7 +6,7 @@
 
 import * as assert from 'assert';
 import * as modes from 'vs/editor/common/modes';
-import {AbstractState} from 'vs/editor/common/modes/abstractState';
+import {AbstractState, ITokenizationResult} from 'vs/editor/common/modes/abstractState';
 import {handleEvent} from 'vs/editor/common/modes/supports';
 import {IModeLocator, ILeavingNestedModeData, TokenizationSupport} from 'vs/editor/common/modes/supports/tokenizationSupport';
 import {createMockLineContext} from 'vs/editor/test/common/modesTestUtils';
@@ -36,7 +36,7 @@ export class StateMemorizingLastWord extends AbstractState {
 		return new StateMemorizingLastWord(this.getModeId(), this.descriptor, this.lastWord);
 	}
 
-	public tokenize(stream:modes.IStream):modes.ITokenizationResult {
+	public tokenize(stream:modes.IStream):ITokenizationResult {
 		stream.setTokenRules('[]{}()==--', '\t \u00a0');
 		if (stream.skipWhitespace() !== '') {
 			return {
@@ -61,7 +61,7 @@ export class SwitchingMode extends MockMode {
 		modes.TokenizationRegistry.register(this.getId(), new TokenizationSupport(null, this.getId(), this, true));
 	}
 
-	public getInitialState():modes.IState {
+	public getInitialState():AbstractState {
 		return new StateMemorizingLastWord(this.getId(), this._switchingModeDescriptor, null);
 	}
 
@@ -167,7 +167,7 @@ suite('Editor Modes - Tokenization', () => {
 				return new State(this.getModeId());
 			}
 
-			public tokenize(stream:modes.IStream):modes.ITokenizationResult {
+			public tokenize(stream:modes.IStream):ITokenizationResult {
 				return { type: stream.next() === '.' ? '' : 'text' };
 			}
 		}