Adopt new ViewLineToken, use model coordinates consistently

src/vs/editor/common/core/viewLineToken.ts

@@ -32,6 +32,10 @@ export class ViewLineToken {
 		return TokenMetadata.getClassNameFromMetadata(this._metadata);
 	}
 
+	public getInlineStyle(colorMap: string[]): string {
+		return TokenMetadata.getInlineStyleFromMetadata(this._metadata, colorMap);
+	}
+
 	private static _equals(a: ViewLineToken, b: ViewLineToken): boolean {
 		return (
 			a.endIndex === b.endIndex

src/vs/editor/common/model/tokensBinaryEncoding.ts

@@ -45,4 +45,21 @@ export class TokenMetadata {
 
 		return className;
 	}
+
+	public static getInlineStyleFromMetadata(metadata: number, colorMap: string[]): string {
+		const foreground = this.getForeground(metadata);
+		const fontStyle = this.getFontStyle(metadata);
+
+		let result = `color: ${colorMap[foreground]};`;
+		if (fontStyle & FontStyle.Italic) {
+			result += 'font-style: italic;';
+		}
+		if (fontStyle & FontStyle.Bold) {
+			result += 'font-weight: bold;';
+		}
+		if (fontStyle & FontStyle.Underline) {
+			result += 'text-decoration: underline;';
+		}
+		return result;
+	}
 }

src/vs/editor/common/modes/textToHtmlTokenizer.ts

@@ -15,25 +15,22 @@ export function tokenizeToString(text: string, languageId: string): string {
 	return _tokenizeToString(text, _getSafeTokenizationSupport(languageId));
 }
 
-export function tokenizeLineToHTML(text: string, viewLineTokens: ViewLineToken[], rules: { [key: string]: string }, options: { startOffset: number, endOffset: number, tabSize: number }): string {
-	let tabSize = options.tabSize;
+export function tokenizeLineToHTML(text: string, viewLineTokens: ViewLineToken[], colorMap: string[], startOffset: number, endOffset: number, tabSize: number): string {
 	let result = `<div>`;
-	let charIndex = options.startOffset;
+	let charIndex = startOffset;
 	let tabsCharDelta = 0;
 
 	for (let tokenIndex = 0, lenJ = viewLineTokens.length; tokenIndex < lenJ; tokenIndex++) {
 		const token = viewLineTokens[tokenIndex];
 		const tokenEndIndex = token.endIndex;
 
-		if (token.endIndex < options.startOffset) {
+		if (token.endIndex <= startOffset) {
 			continue;
 		}
 
-		const tokenType = token.getType();
-		let partContentCnt = 0;
 		let partContent = '';
 
-		for (; charIndex < tokenEndIndex && charIndex < options.endOffset; charIndex++) {
+		for (; charIndex < tokenEndIndex && charIndex < endOffset; charIndex++) {
 			const charCode = text.charCodeAt(charIndex);
 
 			switch (charCode) {
@@ -42,59 +39,48 @@ export function tokenizeLineToHTML(text: string, viewLineTokens: ViewLineToken[]
 					tabsCharDelta += insertSpacesCount - 1;
 					while (insertSpacesCount > 0) {
 						partContent += '&nbsp;';
-						partContentCnt++;
 						insertSpacesCount--;
 					}
 					break;
 
 				case CharCode.Space:
 					partContent += '&nbsp;';
-					partContentCnt++;
 					break;
 
 				case CharCode.LessThan:
 					partContent += '&lt;';
-					partContentCnt++;
 					break;
 
 				case CharCode.GreaterThan:
 					partContent += '&gt;';
-					partContentCnt++;
 					break;
 
 				case CharCode.Ampersand:
 					partContent += '&amp;';
-					partContentCnt++;
 					break;
 
 				case CharCode.Null:
 					partContent += '&#00;';
-					partContentCnt++;
 					break;
 
 				case CharCode.UTF8_BOM:
 				case CharCode.LINE_SEPARATOR_2028:
 					partContent += '\ufffd';
-					partContentCnt++;
 					break;
 
 				case CharCode.CarriageReturn:
 					// zero width space, because carriage return would introduce a line break
 					partContent += '&#8203';
-					partContentCnt++;
 					break;
 
 				default:
 					partContent += String.fromCharCode(charCode);
-					partContentCnt++;
 			}
 		}
 
-		// TODO: adopt new view line tokens.
-		let style = tokenType.split(' ').map(type => rules[type]).join('');
-		result += `<span style="${style}">${partContent}</span>`;
+		result += `<span style="${token.getInlineStyle(colorMap)}">${partContent}</span>`;
 
-		if (token.endIndex > options.endOffset) {
+		if (token.endIndex > endOffset || charIndex >= endOffset) {
 			break;
 		}
 	}

src/vs/editor/common/viewModel/viewModelImpl.ts

@@ -11,7 +11,7 @@ import { Position } from 'vs/editor/common/core/position';
 import { Range } from 'vs/editor/common/core/range';
 import { Selection } from 'vs/editor/common/core/selection';
 import * as editorCommon from 'vs/editor/common/editorCommon';
-import { TokenizationRegistry } from 'vs/editor/common/modes';
+import { TokenizationRegistry, ColorId } from 'vs/editor/common/modes';
 import { tokenizeLineToHTML } from 'vs/editor/common/modes/textToHtmlTokenizer';
 import { ViewModelCursors } from 'vs/editor/common/viewModel/viewModelCursors';
 import { ViewModelDecorations } from 'vs/editor/common/viewModel/viewModelDecorations';
@@ -595,69 +595,67 @@ export class ViewModel extends EventEmitter implements IViewModel {
 		}
 	}
 
-	public getHTMLToCopy(ranges: Range[], enableEmptySelectionClipboard: boolean): string {
-		// TODO: adopt new view line tokens.
-		let rules: { [key: string]: string } = {};
-		let colorMap = TokenizationRegistry.getColorMap();
-		for (let i = 1, len = colorMap.length; i < len; i++) {
-			let color = colorMap[i];
-			rules[`mtk${i}`] = `color: ${color.toRGBHex()};`;
+	public getHTMLToCopy(viewRanges: Range[], enableEmptySelectionClipboard: boolean): string {
+		if (viewRanges.length !== 1) {
+			// no multiple selection support at this time
+			return null;
 		}
-		rules['mtki'] = 'font-style: italic;';
-		rules['mtkb'] = 'font-weight: bold;';
-		rules['mtku'] = 'text-decoration: underline;';
 
-		let defaultForegroundColor = colorMap[1].toRGBHex();
-		let defaultBackgroundColor = colorMap[2].toRGBHex();
+		let range = this.coordinatesConverter.convertViewRangeToModelRange(viewRanges[0]);
+		if (range.isEmpty()) {
+			if (!enableEmptySelectionClipboard) {
+				// nothing to copy
+				return null;
+			}
+			let lineNumber = range.startLineNumber;
+			range = new Range(lineNumber, this.model.getLineMinColumn(lineNumber), lineNumber, this.model.getLineMaxColumn(lineNumber));
+		}
 
-		let fontInfo = this.configuration.editor.fontInfo;
+		const fontInfo = this.configuration.editor.fontInfo;
+		const colorMap = this._getColorMap();
+
+		return (
+			`<div style="`
+			+ `color: ${colorMap[ColorId.DefaultForeground]};`
+			+ `background-color: ${colorMap[ColorId.DefaultBackground]};`
+			+ `font-family: ${fontInfo.fontFamily};`
+			+ `font-weight: ${fontInfo.fontWeight};`
+			+ `font-size: ${fontInfo.fontSize}px;`
+			+ `line-height: ${fontInfo.lineHeight}px`
+			+ `">`
+			+ this._getHTMLToCopy(range, colorMap)
+			+ '</div>'
+		);
+	}
 
-		let output = `<div style="color: ${defaultForegroundColor}; background-color: ${defaultBackgroundColor};` +
-			`font-family: ${fontInfo.fontFamily}; font-weight: ${fontInfo.fontWeight}; font-size: ${fontInfo.fontSize}px; line-height: ${fontInfo.lineHeight}px">`;
+	private _getHTMLToCopy(modelRange: Range, colorMap: string[]): string {
+		const startLineNumber = modelRange.startLineNumber;
+		const startColumn = modelRange.startColumn;
+		const endLineNumber = modelRange.endLineNumber;
+		const endColumn = modelRange.endColumn;
 
-		if (ranges.length === 1) {
-			let range: Range = ranges[0];
+		const tabSize = this.getTabSize();
 
-			if (range.isEmpty()) {
-				if (enableEmptySelectionClipboard) {
-					let modelLineNumber = this.coordinatesConverter.convertViewPositionToModelPosition(new Position(range.startLineNumber, 1)).lineNumber;
-					let viewLineStart = new Position(range.startLineNumber, 1);
-					let viewLineEnd = new Position(range.startLineNumber, this.getLineMaxColumn(range.startLineNumber));
-					let startOffset = this.coordinatesConverter.convertViewPositionToModelPosition(viewLineStart).column - 1;
-					let endOffset = this.coordinatesConverter.convertViewPositionToModelPosition(viewLineEnd).column - 1;
-					let viewLineRenderingData = this.getViewLineRenderingData(new Range(viewLineStart.lineNumber, viewLineStart.column, viewLineEnd.lineNumber, viewLineEnd.column), modelLineNumber);
-					let html = tokenizeLineToHTML(this.getModelLineContent(modelLineNumber),
-						viewLineRenderingData.tokens,
-						rules,
-						{
-							startOffset: startOffset,
-							endOffset: endOffset,
-							tabSize: this.getTabSize()
-						});
-					output += `${html}`;
-				} else {
-					return '';
-				}
-			} else {
-				for (let i = 0, lineCount = range.endLineNumber - range.startLineNumber; i <= lineCount; i++) {
-					let viewLineRenderingData = this.getViewLineRenderingData(range, range.startLineNumber + i);
-					let lineContent = viewLineRenderingData.content;
-					let startOffset = i === 0 ? range.startColumn - 1 : 0;
-					let endOffset = i === lineCount ? range.endColumn - 1 : lineContent.length;
-
-					let html = tokenizeLineToHTML(lineContent, viewLineRenderingData.tokens, rules,
-						{
-							startOffset: startOffset,
-							endOffset: endOffset,
-							tabSize: this.getTabSize()
-						});
-					output += `${html}`;
-				}
-			}
+		let result = '';
+
+		for (let lineNumber = startLineNumber; lineNumber <= endLineNumber; lineNumber++) {
+			const lineTokens = this.model.getLineTokens(lineNumber, true);
+			const lineContent = lineTokens.getLineContent();
+			const startOffset = (lineNumber === startLineNumber ? startColumn - 1 : 0);
+			const endOffset = (lineNumber === endLineNumber ? endColumn - 1 : lineContent.length);
+
+			result += tokenizeLineToHTML(lineContent, lineTokens.inflate(), colorMap, startOffset, endOffset, tabSize);
 		}
 
-		output += '</div>';
+		return result;
+	}
 
-		return output;
+	private _getColorMap(): string[] {
+		let colorMap = TokenizationRegistry.getColorMap();
+		let result: string[] = [null];
+		for (let i = 1, len = colorMap.length; i < len; i++) {
+			result[i] = colorMap[i].toRGBHex();
+		}
+		return result;
 	}
 }

src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts

@@ -5,10 +5,11 @@
 'use strict';
 
 import * as assert from 'assert';
-import { TokenizationRegistry, IState, LanguageIdentifier, ColorId, MetadataConsts } from 'vs/editor/common/modes';
-import { tokenizeToString } from 'vs/editor/common/modes/textToHtmlTokenizer';
+import { TokenizationRegistry, IState, LanguageIdentifier, ColorId, FontStyle, MetadataConsts } from 'vs/editor/common/modes';
+import { tokenizeToString, tokenizeLineToHTML } from 'vs/editor/common/modes/textToHtmlTokenizer';
 import { MockMode } from 'vs/editor/test/common/mocks/mockMode';
 import { TokenizationResult2 } from 'vs/editor/common/core/token';
+import { ViewLineToken } from 'vs/editor/common/core/viewLineToken';
 
 suite('Editor Modes - textToHtmlTokenizer', () => {
 	function toStr(pieces: { className: string; text: string }[]): string {
@@ -64,6 +65,134 @@ suite('Editor Modes - textToHtmlTokenizer', () => {
 		mode.dispose();
 	});
 
+	test('tokenizeLineToHTML', () => {
+		const text = 'Ciao hello world!';
+		const lineTokens = [
+			new ViewLineToken(
+				4,
+				(
+					(3 << MetadataConsts.FOREGROUND_OFFSET)
+					| ((FontStyle.Bold | FontStyle.Italic) << MetadataConsts.FONT_STYLE_OFFSET)
+				) >>> 0
+			),
+			new ViewLineToken(
+				5,
+				(
+					(1 << MetadataConsts.FOREGROUND_OFFSET)
+				) >>> 0
+			),
+			new ViewLineToken(
+				10,
+				(
+					(4 << MetadataConsts.FOREGROUND_OFFSET)
+				) >>> 0
+			),
+			new ViewLineToken(
+				11,
+				(
+					(1 << MetadataConsts.FOREGROUND_OFFSET)
+				) >>> 0
+			),
+			new ViewLineToken(
+				17,
+				(
+					(5 << MetadataConsts.FOREGROUND_OFFSET)
+					| ((FontStyle.Underline) << MetadataConsts.FONT_STYLE_OFFSET)
+				) >>> 0
+			)
+		];
+		const colorMap = [null, '#000000', '#ffffff', '#ff0000', '#00ff00', '#0000ff'];
+
+		assert.equal(
+			tokenizeLineToHTML(text, lineTokens, colorMap, 0, 17, 4),
+			[
+				'<div>',
+				'<span style="color: #ff0000;font-style: italic;font-weight: bold;">Ciao</span>',
+				'<span style="color: #000000;">&nbsp;</span>',
+				'<span style="color: #00ff00;">hello</span>',
+				'<span style="color: #000000;">&nbsp;</span>',
+				'<span style="color: #0000ff;text-decoration: underline;">world!</span>',
+				'</div>'
+			].join('')
+		);
+
+		assert.equal(
+			tokenizeLineToHTML(text, lineTokens, colorMap, 0, 12, 4),
+			[
+				'<div>',
+				'<span style="color: #ff0000;font-style: italic;font-weight: bold;">Ciao</span>',
+				'<span style="color: #000000;">&nbsp;</span>',
+				'<span style="color: #00ff00;">hello</span>',
+				'<span style="color: #000000;">&nbsp;</span>',
+				'<span style="color: #0000ff;text-decoration: underline;">w</span>',
+				'</div>'
+			].join('')
+		);
+
+		assert.equal(
+			tokenizeLineToHTML(text, lineTokens, colorMap, 0, 11, 4),
+			[
+				'<div>',
+				'<span style="color: #ff0000;font-style: italic;font-weight: bold;">Ciao</span>',
+				'<span style="color: #000000;">&nbsp;</span>',
+				'<span style="color: #00ff00;">hello</span>',
+				'<span style="color: #000000;">&nbsp;</span>',
+				'</div>'
+			].join('')
+		);
+
+		assert.equal(
+			tokenizeLineToHTML(text, lineTokens, colorMap, 1, 11, 4),
+			[
+				'<div>',
+				'<span style="color: #ff0000;font-style: italic;font-weight: bold;">iao</span>',
+				'<span style="color: #000000;">&nbsp;</span>',
+				'<span style="color: #00ff00;">hello</span>',
+				'<span style="color: #000000;">&nbsp;</span>',
+				'</div>'
+			].join('')
+		);
+
+		assert.equal(
+			tokenizeLineToHTML(text, lineTokens, colorMap, 4, 11, 4),
+			[
+				'<div>',
+				'<span style="color: #000000;">&nbsp;</span>',
+				'<span style="color: #00ff00;">hello</span>',
+				'<span style="color: #000000;">&nbsp;</span>',
+				'</div>'
+			].join('')
+		);
+
+		assert.equal(
+			tokenizeLineToHTML(text, lineTokens, colorMap, 5, 11, 4),
+			[
+				'<div>',
+				'<span style="color: #00ff00;">hello</span>',
+				'<span style="color: #000000;">&nbsp;</span>',
+				'</div>'
+			].join('')
+		);
+
+		assert.equal(
+			tokenizeLineToHTML(text, lineTokens, colorMap, 5, 10, 4),
+			[
+				'<div>',
+				'<span style="color: #00ff00;">hello</span>',
+				'</div>'
+			].join('')
+		);
+
+		assert.equal(
+			tokenizeLineToHTML(text, lineTokens, colorMap, 6, 9, 4),
+			[
+				'<div>',
+				'<span style="color: #00ff00;">ell</span>',
+				'</div>'
+			].join('')
+		);
+	});
+
 });
 
 class Mode extends MockMode {