Remove ModeTransition

build/monaco/monaco.d.ts.recipe

@@ -48,11 +48,12 @@ declare module monaco {
 #include(vs/editor/common/core/position): Position
 #include(vs/editor/common/core/range): Range
 #include(vs/editor/common/core/selection): Selection, SelectionDirection
+#include(vs/editor/common/core/token): Token
 }
 
 declare module monaco.editor {
 
-#includeAll(vs/editor/browser/standalone/standaloneEditor;modes.=>languages.):
+#includeAll(vs/editor/browser/standalone/standaloneEditor;modes.=>languages.;editorCommon.=>):
 #include(vs/editor/common/services/webWorker): MonacoWebWorker, IWebWorkerOptions
 #include(vs/editor/browser/standalone/standaloneCodeEditor): IEditorConstructionOptions, IDiffEditorConstructionOptions, IStandaloneCodeEditor, IStandaloneDiffEditor
 export interface ICommandHandler {

src/vs/editor/browser/standalone/standaloneEditor.ts

@@ -6,9 +6,6 @@
 
 import 'vs/css!./media/standalone-tokens';
 import * as editorCommon from 'vs/editor/common/editorCommon';
-/* tslint:disable:duplicate-imports */
-import { IModel } from 'vs/editor/common/editorCommon';
-/* tslint:disable:duplicate-imports */
 import { ContentWidgetPositionPreference, OverlayWidgetPositionPreference } from 'vs/editor/browser/editorBrowser';
 import { StandaloneEditor, IStandaloneCodeEditor, StandaloneDiffEditor, IStandaloneDiffEditor, IEditorConstructionOptions, IDiffEditorConstructionOptions } from 'vs/editor/browser/standalone/standaloneCodeEditor';
 import { ScrollbarVisibility } from 'vs/base/common/scrollable';
@@ -33,9 +30,9 @@ import { IContextKeyService } from 'vs/platform/contextkey/common/contextkey';
 import { ICodeEditorService } from 'vs/editor/common/services/codeEditorService';
 import { IEditorWorkerService } from 'vs/editor/common/services/editorWorkerService';
 import { ITextModelResolverService } from 'vs/editor/common/services/resolverService';
-import { IState, ITokenizationSupport, TokenizationRegistry } from 'vs/editor/common/modes';
 import { NULL_STATE, nullTokenize } from 'vs/editor/common/modes/nullMode';
 import { IStandaloneColorService } from 'vs/editor/common/services/standaloneColorService';
+import { Token } from 'vs/editor/common/core/token';
 
 /**
  * @internal
@@ -136,7 +133,7 @@ export function createDiffNavigator(diffEditor: IStandaloneDiffEditor, opts?: ID
 	return new DiffNavigator(diffEditor, opts);
 }
 
-function doCreateModel(value: string, mode: TPromise<modes.IMode>, uri?: URI): IModel {
+function doCreateModel(value: string, mode: TPromise<modes.IMode>, uri?: URI): editorCommon.IModel {
 	return StaticServices.modelService.get().createModel(value, mode, uri);
 }
 
@@ -144,7 +141,7 @@ function doCreateModel(value: string, mode: TPromise<modes.IMode>, uri?: URI): I
  * Create a new editor model.
  * You can specify the language that should be set for this model or let the language be inferred from the `uri`.
  */
-export function createModel(value: string, language?: string, uri?: URI): IModel {
+export function createModel(value: string, language?: string, uri?: URI): editorCommon.IModel {
 	value = value || '';
 
 	if (!language) {
@@ -164,14 +161,14 @@ export function createModel(value: string, language?: string, uri?: URI): IModel
 /**
  * Change the language for a model.
  */
-export function setModelLanguage(model: IModel, language: string): void {
+export function setModelLanguage(model: editorCommon.IModel, language: string): void {
 	StaticServices.modelService.get().setMode(model, StaticServices.modeService.get().getOrCreateMode(language));
 }
 
 /**
  * Set the markers for a model.
  */
-export function setModelMarkers(model: IModel, owner: string, markers: IMarkerData[]): void {
+export function setModelMarkers(model: editorCommon.IModel, owner: string, markers: IMarkerData[]): void {
 	if (model) {
 		StaticServices.markerService.get().changeOne(owner, model.uri, markers);
 	}
@@ -180,14 +177,14 @@ export function setModelMarkers(model: IModel, owner: string, markers: IMarkerDa
 /**
  * Get the model that has `uri` if it exists.
  */
-export function getModel(uri: URI): IModel {
+export function getModel(uri: URI): editorCommon.IModel {
 	return StaticServices.modelService.get().getModel(uri);
 }
 
 /**
  * Get all the created models.
  */
-export function getModels(): IModel[] {
+export function getModels(): editorCommon.IModel[] {
 	return StaticServices.modelService.get().getModels();
 }
 
@@ -195,7 +192,7 @@ export function getModels(): IModel[] {
  * Emitted when a model is created.
  * @event
  */
-export function onDidCreateModel(listener: (model: IModel) => void): IDisposable {
+export function onDidCreateModel(listener: (model: editorCommon.IModel) => void): IDisposable {
 	return StaticServices.modelService.get().onModelAdded(listener);
 }
 
@@ -203,7 +200,7 @@ export function onDidCreateModel(listener: (model: IModel) => void): IDisposable
  * Emitted right before a model is disposed.
  * @event
  */
-export function onWillDisposeModel(listener: (model: IModel) => void): IDisposable {
+export function onWillDisposeModel(listener: (model: editorCommon.IModel) => void): IDisposable {
 	return StaticServices.modelService.get().onModelRemoved(listener);
 }
 
@@ -211,7 +208,7 @@ export function onWillDisposeModel(listener: (model: IModel) => void): IDisposab
  * Emitted when a different language is set to a model.
  * @event
  */
-export function onDidChangeModelLanguage(listener: (e: { readonly model: IModel; readonly oldLanguage: string; }) => void): IDisposable {
+export function onDidChangeModelLanguage(listener: (e: { readonly model: editorCommon.IModel; readonly oldLanguage: string; }) => void): IDisposable {
 	return StaticServices.modelService.get().onModelModeChanged((e) => {
 		listener({
 			model: e.model,
@@ -255,31 +252,21 @@ export function colorize(text: string, languageId: string, options: IColorizerOp
 /**
  * Colorize a line in a model.
  */
-export function colorizeModelLine(model: IModel, lineNumber: number, tabSize: number = 4): string {
+export function colorizeModelLine(model: editorCommon.IModel, lineNumber: number, tabSize: number = 4): string {
 	return Colorizer.colorizeModelLine(model, lineNumber, tabSize);
 }
 
-export class Token {
-	public readonly offset: number;
-	public readonly type: string;
-
-	constructor(offset: number, type: string) {
-		this.offset = offset;
-		this.type = type;
-	}
-}
-
 /**
  * @internal
  */
-function getSafeTokenizationSupport(languageId: string): ITokenizationSupport {
-	let tokenizationSupport = TokenizationRegistry.get(languageId);
+function getSafeTokenizationSupport(languageId: string): modes.ITokenizationSupport {
+	let tokenizationSupport = modes.TokenizationRegistry.get(languageId);
 	if (tokenizationSupport) {
 		return tokenizationSupport;
 	}
 	return {
 		getInitialState: () => NULL_STATE,
-		tokenize: (line: string, state: IState, deltaOffset: number) => nullTokenize(languageId, line, state, deltaOffset),
+		tokenize: (line: string, state: modes.IState, deltaOffset: number) => nullTokenize(languageId, line, state, deltaOffset),
 		tokenize3: undefined,
 	};
 }
@@ -300,7 +287,7 @@ export function tokenize(text: string, languageId: string): Token[][] {
 		let line = lines[i];
 		let tokenizationResult = tokenizationSupport.tokenize(line, state, 0);
 
-		result[i] = tokenizationResult.tokens.map((t) => new Token(t.startIndex, t.type));
+		result[i] = tokenizationResult.tokens;
 		state = tokenizationResult.endState;
 	}
 	return result;
@@ -358,7 +345,6 @@ export function createMonacoEditorAPI(): typeof monaco.editor {
 		BareFontInfo: <any>editorCommon.BareFontInfo,
 		FontInfo: <any>editorCommon.FontInfo,
 		TextModelResolvedOptions: <any>editorCommon.TextModelResolvedOptions,
-		Token: Token,
 
 		// vars
 		EditorType: editorCommon.EditorType,

src/vs/editor/browser/standalone/standaloneLanguages.ts

@@ -24,7 +24,6 @@ import { createTokenizationSupport } from 'vs/editor/common/modes/monarch/monarc
 import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
 import { IMarkerData } from 'vs/platform/markers/common/markers';
 import { Token } from 'vs/editor/common/core/token';
-import { ModeTransition } from 'vs/editor/common/core/modeTransition';
 import { IStandaloneColorService } from 'vs/editor/common/services/standaloneColorService';
 
 /**
@@ -90,18 +89,18 @@ export class TokenizationSupport2Adapter implements modes.ITokenizationSupport {
 		return this._actual.getInitialState();
 	}
 
-	private _toClassicTokens(tokens: modes.IToken2[], offsetDelta: number): Token[] {
+	private _toClassicTokens(tokens: modes.IToken2[], language: string, offsetDelta: number): Token[] {
 		let result: Token[] = [];
 		for (let i = 0, len = tokens.length; i < len; i++) {
 			let t = tokens[i];
-			result[i] = new Token(t.startIndex + offsetDelta, t.scopes);
+			result[i] = new Token(t.startIndex + offsetDelta, t.scopes, language);
 		}
 		return result;
 	}
 
 	public tokenize(line: string, state: modes.IState, offsetDelta: number): modes.ILineTokens {
 		let actualResult = this._actual.tokenize(line, state);
-		let tokens = this._toClassicTokens(actualResult.tokens, offsetDelta);
+		let tokens = this._toClassicTokens(actualResult.tokens, this._languageIdentifier.sid, offsetDelta);
 
 		let endState: modes.IState;
 		// try to save an object if possible
@@ -113,8 +112,7 @@ export class TokenizationSupport2Adapter implements modes.ITokenizationSupport {
 
 		return {
 			tokens: tokens,
-			endState: endState,
-			modeTransitions: [new ModeTransition(offsetDelta, this._languageIdentifier.sid)],
+			endState: endState
 		};
 	}
 

src/vs/editor/common/core/modeTransition.ts

-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the MIT License. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-'use strict';
-
-import { Arrays } from 'vs/editor/common/core/arrays';
-
-export class ModeTransition {
-	_modeTransitionBrand: void;
-
-	public readonly startIndex: number;
-	public readonly modeId: string;
-
-	constructor(startIndex: number, modeId: string) {
-		this.startIndex = startIndex | 0;
-		this.modeId = modeId;
-	}
-
-	public static findIndexInSegmentsArray(arr: ModeTransition[], desiredIndex: number): number {
-		return Arrays.findIndexInSegmentsArray(arr, desiredIndex);
-	}
-}

src/vs/editor/common/core/token.ts

@@ -4,45 +4,20 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
-import { Arrays } from 'vs/editor/common/core/arrays';
-
 export class Token {
 	_tokenBrand: void;
 
-	public readonly startIndex: number;
+	public readonly offset: number;
 	public readonly type: string;
+	public readonly language: string;
 
-	constructor(startIndex: number, type: string) {
-		this.startIndex = startIndex | 0;// @perf
+	constructor(offset: number, type: string, language: string) {
+		this.offset = offset | 0;// @perf
 		this.type = type;
+		this.language = language;
 	}
 
 	public toString(): string {
-		return '(' + this.startIndex + ', ' + this.type + ')';
-	}
-
-	public equals(other: Token): boolean {
-		return (
-			this.startIndex === other.startIndex
-			&& this.type === other.type
-		);
-	}
-
-	public static findIndexInSegmentsArray(arr: Token[], desiredIndex: number): number {
-		return Arrays.findIndexInSegmentsArray(arr, desiredIndex);
-	}
-
-	public static equalsArray(a: Token[], b: Token[]): boolean {
-		let aLen = a.length;
-		let bLen = b.length;
-		if (aLen !== bLen) {
-			return false;
-		}
-		for (let i = 0; i < aLen; i++) {
-			if (!a[i].equals(b[i])) {
-				return false;
-			}
-		}
-		return true;
+		return '(' + this.offset + ', ' + this.type + ')';
 	}
 }

src/vs/editor/common/modes.ts

@@ -9,7 +9,6 @@ import { IDisposable } from 'vs/base/common/lifecycle';
 import URI from 'vs/base/common/uri';
 import { IFilter } from 'vs/base/common/filters';
 import * as editorCommon from 'vs/editor/common/editorCommon';
-import { ModeTransition } from 'vs/editor/common/core/modeTransition';
 import { Token } from 'vs/editor/common/core/token';
 import LanguageFeatureRegistry from 'vs/editor/common/modes/languageFeatureRegistry';
 import { CancellationToken } from 'vs/base/common/cancellation';
@@ -55,7 +54,6 @@ export interface IMode {
 export interface ILineTokens {
 	tokens: Token[];
 	endState: IState;
-	modeTransitions: ModeTransition[];
 }
 
 /**

src/vs/editor/common/modes/monarch/monarchLexer.ts

@@ -15,7 +15,6 @@ import * as monarchCommon from 'vs/editor/common/modes/monarch/monarchCommon';
 import { IModeService } from 'vs/editor/common/services/modeService';
 import { Token } from 'vs/editor/common/core/token';
 import { NULL_STATE, NULL_MODE_ID } from 'vs/editor/common/modes/nullMode';
-import { ModeTransition } from 'vs/editor/common/core/modeTransition';
 import { IStandaloneColorService } from 'vs/editor/common/services/standaloneColorService';
 import { Theme } from 'vs/editor/common/modes/supports/tokenization';
 
@@ -240,35 +239,29 @@ interface IMonarchTokensCollector {
 
 class MonarchClassicTokensCollector implements IMonarchTokensCollector {
 
-	private _modeTransitions: ModeTransition[];
 	private _tokens: Token[];
-
-	private _lastModeId: string;
+	private _language: string;
 	private _lastTokenType: string;
+	private _lastTokenLanguage: string;
 
 	constructor() {
-		this._modeTransitions = [];
 		this._tokens = [];
-
-		this._lastModeId = null;
+		this._language = null;
 		this._lastTokenType = null;
+		this._lastTokenLanguage = null;
 	}
 
 	public enterMode(startOffset: number, modeId: string): void {
-		if (this._lastModeId === modeId) {
-			// Avoid transitioning to the same mode (this can happen in case of empty embedded modes)
-			return;
-		}
-		this._lastModeId = modeId;
-		this._modeTransitions.push(new ModeTransition(startOffset, modeId));
+		this._language = modeId;
 	}
 
 	public emit(startOffset: number, type: string): void {
-		if (this._lastTokenType === type) {
+		if (this._lastTokenType === type && this._lastTokenLanguage === this._language) {
 			return;
 		}
 		this._lastTokenType = type;
-		this._tokens.push(new Token(startOffset, type));
+		this._lastTokenLanguage = this._language;
+		this._tokens.push(new Token(startOffset, type, this._language));
 	}
 
 	public nestedModeTokenize(embeddedModeLine: string, embeddedModeData: EmbeddedModeData, offsetDelta: number): modes.IState {
@@ -285,15 +278,14 @@ class MonarchClassicTokensCollector implements IMonarchTokensCollector {
 		let nestedResult = nestedModeTokenizationSupport.tokenize(embeddedModeLine, embeddedModeState, offsetDelta);
 		this._tokens = this._tokens.concat(nestedResult.tokens);
 		this._lastTokenType = null;
-		this._modeTransitions = this._modeTransitions.concat(nestedResult.modeTransitions);
-		this._lastModeId = null;
+		this._lastTokenLanguage = null;
+		this._language = null;
 		return nestedResult.endState;
 	}
 
 	public finalize(endState: MonarchLineState): modes.ILineTokens {
 		return {
 			tokens: this._tokens,
-			modeTransitions: this._modeTransitions,
 			endState: endState
 		};
 	}

src/vs/editor/common/modes/nullMode.ts

@@ -5,7 +5,6 @@
 'use strict';
 
 import { IState, ILineTokens, ILineTokens3, ColorId, MetadataConsts, LanguageIdentifier, FontStyle, StandardTokenType, LanguageId } from 'vs/editor/common/modes';
-import { ModeTransition } from 'vs/editor/common/core/modeTransition';
 import { Token } from 'vs/editor/common/core/token';
 
 class NullStateImpl implements IState {
@@ -26,14 +25,9 @@ export const NULL_MODE_ID = 'vs.editor.nullMode';
 export const NULL_LANGUAGE_IDENTIFIER = new LanguageIdentifier(NULL_MODE_ID, LanguageId.Null);
 
 export function nullTokenize(modeId: string, buffer: string, state: IState, deltaOffset: number): ILineTokens {
-	let tokens: Token[] = [new Token(deltaOffset, '')];
-
-	let modeTransitions: ModeTransition[] = [new ModeTransition(deltaOffset, modeId)];
-
 	return {
-		tokens: tokens,
-		endState: state,
-		modeTransitions: modeTransitions
+		tokens: [new Token(deltaOffset, '', modeId)],
+		endState: state
 	};
 }
 

src/vs/editor/common/modes/supports.ts

@@ -5,7 +5,6 @@
 'use strict';
 
 import * as modes from 'vs/editor/common/modes';
-import { ModeTransition } from 'vs/editor/common/core/modeTransition';
 import { Token } from 'vs/editor/common/core/token';
 import { LineTokens } from 'vs/editor/common/core/lineTokens';
 
@@ -13,12 +12,10 @@ export class RawLineTokens implements modes.ILineTokens {
 	_lineTokensBrand: void;
 
 	tokens: Token[];
-	modeTransitions: ModeTransition[];
 	endState: modes.IState;
 
-	constructor(tokens: Token[], modeTransitions: ModeTransition[], endState: modes.IState) {
+	constructor(tokens: Token[], endState: modes.IState) {
 		this.tokens = tokens;
-		this.modeTransitions = modeTransitions;
 		this.endState = endState;
 	}
 }

src/vs/editor/common/standalone/standaloneBase.ts

@@ -11,6 +11,7 @@ import { Range } from 'vs/editor/common/core/range';
 import { Selection, SelectionDirection } from 'vs/editor/common/core/selection';
 import { TPromise } from 'vs/base/common/winjs.base';
 import { CancellationTokenSource } from 'vs/base/common/cancellation';
+import { Token } from 'vs/editor/common/core/token';
 import URI from 'vs/base/common/uri';
 
 // --------------------------------------------
@@ -230,6 +231,7 @@ export function createMonacoBaseAPI(): typeof monaco {
 		SelectionDirection: SelectionDirection,
 		Severity: Severity,
 		Promise: TPromise,
-		Uri: URI
+		Uri: URI,
+		Token: Token
 	};
 }

src/vs/editor/node/textMate/TMSyntax.ts

@@ -16,7 +16,6 @@ import { TMState } from 'vs/editor/node/textMate/TMState';
 import { RawLineTokens } from 'vs/editor/common/modes/supports';
 import { IModeService } from 'vs/editor/common/services/modeService';
 import { IGrammar, Registry, StackElement, IToken, IEmbeddedLanguagesMap as IEmbeddedLanguagesMap2 } from 'vscode-textmate';
-import { ModeTransition } from 'vs/editor/common/core/modeTransition';
 import { Token } from 'vs/editor/common/core/token';
 import { languagesExtPoint } from 'vs/editor/common/services/modeServiceImpl';
 import { IThemeService } from 'vs/workbench/services/themes/common/themeService';
@@ -530,8 +529,7 @@ class Tokenizer {
 		if (line.length >= 20000) {
 			console.log(`Line (${line.substr(0, 15)}...): longer than 20k characters, tokenization skipped.`);
 			return new RawLineTokens(
-				[new Token(offsetDelta, '')],
-				[new ModeTransition(offsetDelta, this._modeId)],
+				[new Token(offsetDelta, '', this._modeId)],
 				state
 			);
 		}
@@ -544,8 +542,7 @@ class Tokenizer {
 				this._stackOverflowReported = true;
 			}
 			return new RawLineTokens(
-				[new Token(offsetDelta, '')],
-				[new ModeTransition(offsetDelta, this._modeId)],
+				[new Token(offsetDelta, '', this._modeId)],
 				state
 			);
 		}
@@ -581,10 +578,9 @@ export function decodeTextMateTokens(topLevelModeId: string, decodeMap: DecodeMa
 
 	// Create the result early and fill in the tokens later
 	let tokens: Token[] = [];
-	let modeTransitions: ModeTransition[] = [];
 
 	let lastTokenType: string = null;
-	let lastModeId: string = null;
+	let lastTokenLanguage: string = null;
 
 	for (let tokenIndex = 0, len = resultTokens.length; tokenIndex < len; tokenIndex++) {
 		let token = resultTokens[tokenIndex];
@@ -601,20 +597,15 @@ export function decodeTextMateTokens(topLevelModeId: string, decodeMap: DecodeMa
 		}
 
 		// do not push a new token if the type is exactly the same (also helps with ligatures)
-		if (tokenType !== lastTokenType) {
-			tokens.push(new Token(tokenStartIndex + offsetDelta, tokenType));
+		if (tokenType !== lastTokenType || tokenModeId !== lastTokenLanguage) {
+			tokens.push(new Token(tokenStartIndex + offsetDelta, tokenType, tokenModeId));
 			lastTokenType = tokenType;
-		}
-
-		if (tokenModeId !== lastModeId) {
-			modeTransitions.push(new ModeTransition(tokenStartIndex + offsetDelta, tokenModeId));
-			lastModeId = tokenModeId;
+			lastTokenLanguage = tokenModeId;
 		}
 	}
 
 	return new RawLineTokens(
 		tokens,
-		modeTransitions,
 		resultState
 	);
 }

src/vs/monaco.d.ts

@@ -751,6 +751,14 @@ declare module monaco {
          */
         RTL = 1,
     }
+    export class Token {
+        _tokenBrand: void;
+        readonly offset: number;
+        readonly type: string;
+        readonly language: string;
+        constructor(offset: number, type: string, language: string);
+        toString(): string;
+    }
 }
 
 declare module monaco.editor {
@@ -853,12 +861,6 @@ declare module monaco.editor {
      */
     export function colorizeModelLine(model: IModel, lineNumber: number, tabSize?: number): string;
 
-    export class Token {
-        readonly offset: number;
-        readonly type: string;
-        constructor(offset: number, type: string);
-    }
-
     /**
      * Tokenize `text` using language `languageId`
      */