/*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ import * as assert from 'assert'; import { TokenizationResult2 } from 'vs/editor/common/core/token'; import { ColorId, FontStyle, IState, LanguageIdentifier, MetadataConsts, TokenizationRegistry } from 'vs/editor/common/modes'; import { tokenizeLineToHTML, tokenizeToString } from 'vs/editor/common/modes/textToHtmlTokenizer'; import { ViewLineToken, ViewLineTokens } from 'vs/editor/test/common/core/viewLineToken'; import { MockMode } from 'vs/editor/test/common/mocks/mockMode'; suite('Editor Modes - textToHtmlTokenizer', () => { function toStr(pieces: { className: string; text: string }[]): string { let resultArr = pieces.map((t) => `${t.text}`); return resultArr.join(''); } test('TextToHtmlTokenizer 1', () => { let mode = new Mode(); let support = TokenizationRegistry.get(mode.getId())!; let actual = tokenizeToString('.abc..def...gh', support); let expected = [ { className: 'mtk7', text: '.' }, { className: 'mtk9', text: 'abc' }, { className: 'mtk7', text: '..' }, { className: 'mtk9', text: 'def' }, { className: 'mtk7', text: '...' }, { className: 'mtk9', text: 'gh' }, ]; let expectedStr = `
${toStr(expected)}
`; assert.equal(actual, expectedStr); mode.dispose(); }); test('TextToHtmlTokenizer 2', () => { let mode = new Mode(); let support = TokenizationRegistry.get(mode.getId())!; let actual = tokenizeToString('.abc..def...gh\n.abc..def...gh', support); let expected1 = [ { className: 'mtk7', text: '.' }, { className: 'mtk9', text: 'abc' }, { className: 'mtk7', text: '..' }, { className: 'mtk9', text: 'def' }, { className: 'mtk7', text: '...' }, { className: 'mtk9', text: 'gh' }, ]; let expected2 = [ { className: 'mtk7', text: '.' }, { className: 'mtk9', text: 'abc' }, { className: 'mtk7', text: '..' }, { className: 'mtk9', text: 'def' }, { className: 'mtk7', text: '...' }, { className: 'mtk9', text: 'gh' }, ]; let expectedStr1 = toStr(expected1); let expectedStr2 = toStr(expected2); let expectedStr = `
${expectedStr1}
${expectedStr2}
`; assert.equal(actual, expectedStr); mode.dispose(); }); test('tokenizeLineToHTML', () => { const text = 'Ciao hello world!'; const lineTokens = new ViewLineTokens([ new ViewLineToken( 4, ( (3 << MetadataConsts.FOREGROUND_OFFSET) | ((FontStyle.Bold | FontStyle.Italic) << MetadataConsts.FONT_STYLE_OFFSET) ) >>> 0 ), new ViewLineToken( 5, ( (1 << MetadataConsts.FOREGROUND_OFFSET) ) >>> 0 ), new ViewLineToken( 10, ( (4 << MetadataConsts.FOREGROUND_OFFSET) ) >>> 0 ), new ViewLineToken( 11, ( (1 << MetadataConsts.FOREGROUND_OFFSET) ) >>> 0 ), new ViewLineToken( 17, ( (5 << MetadataConsts.FOREGROUND_OFFSET) | ((FontStyle.Underline) << MetadataConsts.FONT_STYLE_OFFSET) ) >>> 0 ) ]); const colorMap = [null!, '#000000', '#ffffff', '#ff0000', '#00ff00', '#0000ff']; assert.equal( tokenizeLineToHTML(text, lineTokens, colorMap, 0, 17, 4, true), [ '
', 'Ciao', ' ', 'hello', ' ', 'world!', '
' ].join('') ); assert.equal( tokenizeLineToHTML(text, lineTokens, colorMap, 0, 12, 4, true), [ '
', 'Ciao', ' ', 'hello', ' ', 'w', '
' ].join('') ); assert.equal( tokenizeLineToHTML(text, lineTokens, colorMap, 0, 11, 4, true), [ '
', 'Ciao', ' ', 'hello', ' ', '
' ].join('') ); assert.equal( tokenizeLineToHTML(text, lineTokens, colorMap, 1, 11, 4, true), [ '
', 'iao', ' ', 'hello', ' ', '
' ].join('') ); assert.equal( tokenizeLineToHTML(text, lineTokens, colorMap, 4, 11, 4, true), [ '
', ' ', 'hello', ' ', '
' ].join('') ); assert.equal( tokenizeLineToHTML(text, lineTokens, colorMap, 5, 11, 4, true), [ '
', 'hello', ' ', '
' ].join('') ); assert.equal( tokenizeLineToHTML(text, lineTokens, colorMap, 5, 10, 4, true), [ '
', 'hello', '
' ].join('') ); assert.equal( tokenizeLineToHTML(text, lineTokens, colorMap, 6, 9, 4, true), [ '
', 'ell', '
' ].join('') ); }); test('tokenizeLineToHTML handle spaces #35954', () => { const text = ' Ciao hello world!'; const lineTokens = new ViewLineTokens([ new ViewLineToken( 2, ( (1 << MetadataConsts.FOREGROUND_OFFSET) ) >>> 0 ), new ViewLineToken( 6, ( (3 << MetadataConsts.FOREGROUND_OFFSET) | ((FontStyle.Bold | FontStyle.Italic) << MetadataConsts.FONT_STYLE_OFFSET) ) >>> 0 ), new ViewLineToken( 9, ( (1 << MetadataConsts.FOREGROUND_OFFSET) ) >>> 0 ), new ViewLineToken( 14, ( (4 << MetadataConsts.FOREGROUND_OFFSET) ) >>> 0 ), new ViewLineToken( 15, ( (1 << MetadataConsts.FOREGROUND_OFFSET) ) >>> 0 ), new ViewLineToken( 21, ( (5 << MetadataConsts.FOREGROUND_OFFSET) | ((FontStyle.Underline) << MetadataConsts.FONT_STYLE_OFFSET) ) >>> 0 ) ]); const colorMap = [null!, '#000000', '#ffffff', '#ff0000', '#00ff00', '#0000ff']; assert.equal( tokenizeLineToHTML(text, lineTokens, colorMap, 0, 21, 4, true), [ '
', '  ', 'Ciao', '   ', 'hello', ' ', 'world!', '
' ].join('') ); assert.equal( tokenizeLineToHTML(text, lineTokens, colorMap, 0, 17, 4, true), [ '
', '  ', 'Ciao', '   ', 'hello', ' ', 'wo', '
' ].join('') ); assert.equal( tokenizeLineToHTML(text, lineTokens, colorMap, 0, 3, 4, true), [ '
', '  ', 'C', '
' ].join('') ); }); }); class Mode extends MockMode { private static readonly _id = new LanguageIdentifier('textToHtmlTokenizerMode', 3); constructor() { super(Mode._id); this._register(TokenizationRegistry.register(this.getId(), { getInitialState: (): IState => null!, tokenize: undefined!, tokenize2: (line: string, hasEOL: boolean, state: IState): TokenizationResult2 => { let tokensArr: number[] = []; let prevColor: ColorId = -1; for (let i = 0; i < line.length; i++) { let colorId = line.charAt(i) === '.' ? 7 : 9; if (prevColor !== colorId) { tokensArr.push(i); tokensArr.push(( colorId << MetadataConsts.FOREGROUND_OFFSET ) >>> 0); } prevColor = colorId; } let tokens = new Uint32Array(tokensArr.length); for (let i = 0; i < tokens.length; i++) { tokens[i] = tokensArr[i]; } return new TokenizationResult2(tokens, null!); } })); } }