tokenIterator uses only standardTokenType

src/vs/editor/common/editorCommon.ts

@@ -11,7 +11,7 @@ import URI from 'vs/base/common/uri';
 import { TPromise } from 'vs/base/common/winjs.base';
 import { ServicesAccessor, IConstructorSignature1 } from 'vs/platform/instantiation/common/instantiation';
 import { IMode } from 'vs/editor/common/modes';
-import { LineTokens } from 'vs/editor/common/core/lineTokens';
+import { LineTokens, StandardTokenType } from 'vs/editor/common/core/lineTokens';
 import { ScrollbarVisibility } from 'vs/base/common/scrollable';
 import { IDisposable } from 'vs/base/common/lifecycle';
 import { Position } from 'vs/editor/common/core/position';
@@ -1295,7 +1295,7 @@ export interface IWordRange {
  * @internal
  */
 export interface ITokenInfo {
-	readonly type: string;
+	readonly standardType: StandardTokenType;
 	readonly lineNumber: number;
 	readonly startColumn: number;
 	readonly endColumn: number;

src/vs/editor/common/model/tokenIterator.ts

@@ -5,24 +5,24 @@
 'use strict';
 
 import * as editorCommon from 'vs/editor/common/editorCommon';
-import { LineToken } from 'vs/editor/common/core/lineTokens';
+import { LineToken, StandardTokenType } from 'vs/editor/common/core/lineTokens';
 import { Position } from 'vs/editor/common/core/position';
 
 class TokenInfo implements editorCommon.ITokenInfo {
 	_tokenInfoBrand: void;
 
-	_actual: LineToken;
-	public lineNumber: number;
-	public startColumn: number;
-	public endColumn: number;
-	public type: string;
+	readonly _actual: LineToken;
+	public readonly lineNumber: number;
+	public readonly startColumn: number;
+	public readonly endColumn: number;
+	public readonly standardType: StandardTokenType;
 
 	constructor(actual: LineToken, lineNumber: number) {
 		this._actual = actual;
 		this.lineNumber = lineNumber;
 		this.startColumn = this._actual.startOffset + 1;
 		this.endColumn = this._actual.endOffset + 1;
-		this.type = this._actual.type;
+		this.standardType = this._actual.standardType;
 	}
 }
 

src/vs/workbench/parts/themes/test/electron-browser/themes.test.contribution.ts

@@ -118,18 +118,22 @@ class Snapper {
 		return this.modeService.getOrCreateModeByFilenameOrFirstLine(fileName).then(mode => {
 			let result: Data[] = [];
 			let model = new TextModelWithTokens([], TextModel.toRawText(content, TextModel.DEFAULT_CREATION_OPTIONS), mode.getId());
-			model.tokenIterator({ lineNumber: 1, column: 1 }, iterator => {
-				while (iterator.hasNext()) {
-					let tokenInfo = iterator.next();
-					let lineNumber = tokenInfo.lineNumber;
-					let content = model.getValueInRange({ startLineNumber: lineNumber, endLineNumber: lineNumber, startColumn: tokenInfo.startColumn, endColumn: tokenInfo.endColumn });
+			for (let lineNumber = 1, lineCount = model.getLineCount(); lineNumber <= lineCount; lineNumber++) {
+				let lineTokens = model.getLineTokens(lineNumber, false);
+				let lineContent = model.getLineContent(lineNumber);
+
+				for (let i = 0, len = lineTokens.getTokenCount(); i < len; i++) {
+					let tokenType = lineTokens.getTokenType(i);
+					let tokenStartOffset = lineTokens.getTokenStartOffset(i);
+					let tokenEndOffset = lineTokens.getTokenEndOffset(i);
+
 					result.push({
-						c: content,
-						t: this.normalizeType(tokenInfo.type),
+						c: lineContent.substring(tokenStartOffset, tokenEndOffset),
+						t: this.normalizeType(tokenType),
 						r: {}
 					});
 				}
-			});
+			}
 			return this.appendThemeInformation(result);
 		});
 	}